@webiny/api-dynamodb-to-elasticsearch 6.0.0-beta.0 → 6.0.0-rc.1

package/NotEnoughRemainingTimeError.js

@@ -1,11 +1,4 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.NotEnoughRemainingTimeError = void 0;
-var _error = require("@webiny/error");
-class NotEnoughRemainingTimeError extends _error.WebinyError {}
-exports.NotEnoughRemainingTimeError = NotEnoughRemainingTimeError;
+import { WebinyError } from "@webiny/error";
+export class NotEnoughRemainingTimeError extends WebinyError {}
 
 //# sourceMappingURL=NotEnoughRemainingTimeError.js.map

package/NotEnoughRemainingTimeError.js.map

@@ -1 +1 @@
-{"version":3,"names":["_error","require","NotEnoughRemainingTimeError","WebinyError","exports"],"sources":["NotEnoughRemainingTimeError.ts"],"sourcesContent":["import { WebinyError } from \"@webiny/error\";\n\nexport class NotEnoughRemainingTimeError extends WebinyError {}\n"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAEO,MAAMC,2BAA2B,SAASC,kBAAW,CAAC;AAAEC,OAAA,CAAAF,2BAAA,GAAAA,2BAAA","ignoreList":[]}
+{"version":3,"names":["WebinyError","NotEnoughRemainingTimeError"],"sources":["NotEnoughRemainingTimeError.ts"],"sourcesContent":["import { WebinyError } from \"@webiny/error\";\n\nexport class NotEnoughRemainingTimeError extends WebinyError {}\n"],"mappings":"AAAA,SAASA,WAAW,QAAQ,eAAe;AAE3C,OAAO,MAAMC,2BAA2B,SAASD,WAAW,CAAC","ignoreList":[]}

package/Operations.d.ts

@@ -0,0 +1,16 @@
+import type { GenericRecord } from "@webiny/api/types.js";
+import type { IDeleteOperationParams, IInsertOperationParams, IModifyOperationParams, IOperations } from "./types.js";
+export declare enum OperationType {
+    INSERT = "INSERT",
+    MODIFY = "MODIFY",
+    REMOVE = "REMOVE"
+}
+export declare class Operations implements IOperations {
+    private _items;
+    get items(): GenericRecord[];
+    get total(): number;
+    clear(): void;
+    insert(params: IInsertOperationParams): void;
+    modify(params: IModifyOperationParams): void;
+    delete(params: IDeleteOperationParams): void;
+}

package/Operations.js

@@ -0,0 +1,39 @@
+export let OperationType = /*#__PURE__*/function (OperationType) {
+  OperationType["INSERT"] = "INSERT";
+  OperationType["MODIFY"] = "MODIFY";
+  OperationType["REMOVE"] = "REMOVE";
+  return OperationType;
+}({});
+export class Operations {
+  _items = [];
+  get items() {
+    return this._items;
+  }
+  get total() {
+    return this.items.length;
+  }
+  clear() {
+    this._items = [];
+  }
+  insert(params) {
+    this.items.push({
+      index: {
+        _id: params.id,
+        _index: params.index
+      }
+    }, params.data);
+  }
+  modify(params) {
+    this.insert(params);
+  }
+  delete(params) {
+    this.items.push({
+      delete: {
+        _id: params.id,
+        _index: params.index
+      }
+    });
+  }
+}
+
+//# sourceMappingURL=Operations.js.map

package/Operations.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["OperationType","Operations","_items","items","total","length","clear","insert","params","push","index","_id","id","_index","data","modify","delete"],"sources":["Operations.ts"],"sourcesContent":["import type { GenericRecord } from \"@webiny/api/types.js\";\nimport type {\n    IDeleteOperationParams,\n    IInsertOperationParams,\n    IModifyOperationParams,\n    IOperations\n} from \"~/types.js\";\n\nexport enum OperationType {\n    INSERT = \"INSERT\",\n    MODIFY = \"MODIFY\",\n    REMOVE = \"REMOVE\"\n}\n\nexport class Operations implements IOperations {\n    private _items: GenericRecord[] = [];\n\n    public get items(): GenericRecord[] {\n        return this._items;\n    }\n\n    public get total(): number {\n        return this.items.length;\n    }\n\n    public clear() {\n        this._items = [];\n    }\n\n    public insert(params: IInsertOperationParams): void {\n        this.items.push(\n            {\n                index: {\n                    _id: params.id,\n                    _index: params.index\n                }\n            },\n            params.data\n        );\n    }\n\n    public modify(params: IModifyOperationParams): void {\n        this.insert(params);\n    }\n\n    public delete(params: IDeleteOperationParams): void {\n        this.items.push({\n            delete: {\n                _id: params.id,\n                _index: params.index\n            }\n        });\n    }\n}\n"],"mappings":"AAQA,WAAYA,aAAa,0BAAbA,aAAa;EAAbA,aAAa;EAAbA,aAAa;EAAbA,aAAa;EAAA,OAAbA,aAAa;AAAA;AAMzB,OAAO,MAAMC,UAAU,CAAwB;EACnCC,MAAM,GAAoB,EAAE;EAEpC,IAAWC,KAAKA,CAAA,EAAoB;IAChC,OAAO,IAAI,CAACD,MAAM;EACtB;EAEA,IAAWE,KAAKA,CAAA,EAAW;IACvB,OAAO,IAAI,CAACD,KAAK,CAACE,MAAM;EAC5B;EAEOC,KAAKA,CAAA,EAAG;IACX,IAAI,CAACJ,MAAM,GAAG,EAAE;EACpB;EAEOK,MAAMA,CAACC,MAA8B,EAAQ;IAChD,IAAI,CAACL,KAAK,CAACM,IAAI,CACX;MACIC,KAAK,EAAE;QACHC,GAAG,EAAEH,MAAM,CAACI,EAAE;QACdC,MAAM,EAAEL,MAAM,CAACE;MACnB;IACJ,CAAC,EACDF,MAAM,CAACM,IACX,CAAC;EACL;EAEOC,MAAMA,CAACP,MAA8B,EAAQ;IAChD,IAAI,CAACD,MAAM,CAACC,MAAM,CAAC;EACvB;EAEOQ,MAAMA,CAACR,MAA8B,EAAQ;IAChD,IAAI,CAACL,KAAK,CAACM,IAAI,CAAC;MACZO,MAAM,EAAE;QACJL,GAAG,EAAEH,MAAM,CAACI,EAAE;QACdC,MAAM,EAAEL,MAAM,CAACE;MACnB;IACJ,CAAC,CAAC;EACN;AACJ","ignoreList":[]}

package/OperationsBuilder.d.ts

@@ -0,0 +1,10 @@
+import type { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from "./types.js";
+import type { ICompressor } from "@webiny/utils/compression/Compressor.js";
+export interface IOperationsBuilderParams {
+    compressor: ICompressor;
+}
+export declare class OperationsBuilder implements IOperationsBuilder {
+    private readonly compressor;
+    constructor(params: IOperationsBuilderParams);
+    build(params: IOperationsBuilderBuildParams): Promise<IOperations>;
+}

package/OperationsBuilder.js

@@ -0,0 +1,94 @@
+import { Operations, OperationType } from "./Operations.js";
+import { unmarshall } from "./marshall.js";
+export class OperationsBuilder {
+  constructor(params) {
+    this.compressor = params.compressor;
+  }
+  async build(params) {
+    const operations = new Operations();
+    for (const record of params.records) {
+      if (!record.dynamodb) {
+        continue;
+      } else if (!record.eventName) {
+        console.error(`Could not get operation from the record, skipping event "${record.eventID}".`);
+        continue;
+      }
+      const keys = unmarshall(record.dynamodb.Keys);
+      if (!keys?.PK || !keys.SK) {
+        console.error(`Could not get keys from the record, skipping event "${record.eventID}".`);
+        continue;
+      }
+      const id = `${keys.PK}:${keys.SK}`;
+
+      /**
+       * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.
+       * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.
+       */
+      if (record.eventName === OperationType.INSERT || record.eventName === OperationType.MODIFY) {
+        const newImage = unmarshall(record.dynamodb.NewImage);
+        /**
+         * If there is no newImage, silently continue to the next operation.
+         */
+        if (!newImage || typeof newImage !== "object" || Object.keys(newImage).length === 0) {
+          continue;
+        }
+        /**
+         * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,
+         * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still
+         * need to process it.
+         */
+        //
+        else if (newImage.ignore === true) {
+          // Nothing to log here, we are skipping the record intentionally.
+          continue;
+        }
+        /**
+         * Also, possibly there is no index?
+         */
+        //
+        else if (!newImage.index) {
+          console.error(`Could not get index from the new image, skipping event "${record.eventID}".`);
+          console.log({
+            newImage
+          });
+          continue;
+        }
+        /**
+         * We must decompress the data that is going into the Elasticsearch.
+         */
+        const data = await this.compressor.decompress(newImage.data);
+        /**
+         * No point in writing null or undefined data into the Elasticsearch.
+         * This might happen on some error while decompressing. We will log it.
+         *
+         * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.
+         * If it is - it is a bug.
+         */
+        if (data === undefined || data === null) {
+          console.error(`Could not get decompressed data, skipping ES operation "${record.eventName}", ID ${id}. Skipping...`);
+          continue;
+        }
+        operations.insert({
+          id,
+          index: newImage.index,
+          data
+        });
+      } else if (record.eventName === OperationType.REMOVE) {
+        const oldImage = unmarshall(record.dynamodb.OldImage);
+        /**
+         * If there is no index found, silently continue to the next operation.
+         */
+        if (!oldImage?.index) {
+          continue;
+        }
+        operations.delete({
+          id,
+          index: oldImage.index
+        });
+      }
+    }
+    return operations;
+  }
+}
+
+//# sourceMappingURL=OperationsBuilder.js.map

package/OperationsBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["Operations","OperationType","unmarshall","OperationsBuilder","constructor","params","compressor","build","operations","record","records","dynamodb","eventName","console","error","eventID","keys","Keys","PK","SK","id","INSERT","MODIFY","newImage","NewImage","Object","length","ignore","index","log","data","decompress","undefined","insert","REMOVE","oldImage","OldImage","delete"],"sources":["OperationsBuilder.ts"],"sourcesContent":["import type { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from \"./types.js\";\nimport { Operations, OperationType } from \"~/Operations.js\";\nimport { unmarshall } from \"~/marshall.js\";\nimport type { ICompressor } from \"@webiny/utils/compression/Compressor.js\";\n\ninterface RecordDynamoDbImage {\n    data: {\n        compression: string;\n        value: string;\n    };\n    ignore?: boolean;\n    index: string;\n}\n\ninterface RecordDynamoDbKeys {\n    PK: string;\n    SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n    compressor: ICompressor;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n    private readonly compressor: ICompressor;\n\n    public constructor(params: IOperationsBuilderParams) {\n        this.compressor = params.compressor;\n    }\n\n    public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n        const operations = new Operations();\n        for (const record of params.records) {\n            if (!record.dynamodb) {\n                continue;\n            } else if (!record.eventName) {\n                console.error(\n                    `Could not get operation from the record, skipping event \"${record.eventID}\".`\n                );\n                continue;\n            }\n\n            const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n            if (!keys?.PK || !keys.SK) {\n                console.error(\n                    `Could not get keys from the record, skipping event \"${record.eventID}\".`\n                );\n                continue;\n            }\n\n            const id = `${keys.PK}:${keys.SK}`;\n\n            /**\n             * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n             * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n             */\n            if (\n                record.eventName === OperationType.INSERT ||\n                record.eventName === OperationType.MODIFY\n            ) {\n                const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n                /**\n                 * If there is no newImage, silently continue to the next operation.\n                 */\n                if (\n                    !newImage ||\n                    typeof newImage !== \"object\" ||\n                    Object.keys(newImage).length === 0\n                ) {\n                    continue;\n                }\n                /**\n                 * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n                 * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n                 * need to process it.\n                 */\n                //\n                else if (newImage.ignore === true) {\n                    // Nothing to log here, we are skipping the record intentionally.\n                    continue;\n                }\n                /**\n                 * Also, possibly there is no index?\n                 */\n                //\n                else if (!newImage.index) {\n                    console.error(\n                        `Could not get index from the new image, skipping event \"${record.eventID}\".`\n                    );\n                    console.log({ newImage });\n                    continue;\n                }\n                /**\n                 * We must decompress the data that is going into the Elasticsearch.\n                 */\n                const data = await this.compressor.decompress(newImage.data);\n                /**\n                 * No point in writing null or undefined data into the Elasticsearch.\n                 * This might happen on some error while decompressing. We will log it.\n                 *\n                 * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n                 * If it is - it is a bug.\n                 */\n                if (data === undefined || data === null) {\n                    console.error(\n                        `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n                    );\n                    continue;\n                }\n\n                operations.insert({\n                    id,\n                    index: newImage.index,\n                    data\n                });\n            } else if (record.eventName === OperationType.REMOVE) {\n                const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n                /**\n                 * If there is no index found, silently continue to the next operation.\n                 */\n                if (!oldImage?.index) {\n                    continue;\n                }\n                operations.delete({\n                    id,\n                    index: oldImage.index\n                });\n            }\n        }\n        return operations;\n    }\n}\n"],"mappings":"AACA,SAASA,UAAU,EAAEC,aAAa;AAClC,SAASC,UAAU;AAqBnB,OAAO,MAAMC,iBAAiB,CAA+B;EAGlDC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,UAAU,GAAGD,MAAM,CAACC,UAAU;EACvC;EAEA,MAAaC,KAAKA,CAACF,MAAqC,EAAwB;IAC5E,MAAMG,UAAU,GAAG,IAAIR,UAAU,CAAC,CAAC;IACnC,KAAK,MAAMS,MAAM,IAAIJ,MAAM,CAACK,OAAO,EAAE;MACjC,IAAI,CAACD,MAAM,CAACE,QAAQ,EAAE;QAClB;MACJ,CAAC,MAAM,IAAI,CAACF,MAAM,CAACG,SAAS,EAAE;QAC1BC,OAAO,CAACC,KAAK,CACT,4DAA4DL,MAAM,CAACM,OAAO,IAC9E,CAAC;QACD;MACJ;MAEA,MAAMC,IAAI,GAAGd,UAAU,CAAqBO,MAAM,CAACE,QAAQ,CAACM,IAAI,CAAC;MACjE,IAAI,CAACD,IAAI,EAAEE,EAAE,IAAI,CAACF,IAAI,CAACG,EAAE,EAAE;QACvBN,OAAO,CAACC,KAAK,CACT,uDAAuDL,MAAM,CAACM,OAAO,IACzE,CAAC;QACD;MACJ;MAEA,MAAMK,EAAE,GAAG,GAAGJ,IAAI,CAACE,EAAE,IAAIF,IAAI,CAACG,EAAE,EAAE;;MAElC;AACZ;AACA;AACA;MACY,IACIV,MAAM,CAACG,SAAS,KAAKX,aAAa,CAACoB,MAAM,IACzCZ,MAAM,CAACG,SAAS,KAAKX,aAAa,CAACqB,MAAM,EAC3C;QACE,MAAMC,QAAQ,GAAGrB,UAAU,CAAsBO,MAAM,CAACE,QAAQ,CAACa,QAAQ,CAAC;QAC1E;AAChB;AACA;QACgB,IACI,CAACD,QAAQ,IACT,OAAOA,QAAQ,KAAK,QAAQ,IAC5BE,MAAM,CAACT,IAAI,CAACO,QAAQ,CAAC,CAACG,MAAM,KAAK,CAAC,EACpC;UACE;QACJ;QACA;AAChB;AACA;AACA;AACA;QACgB;QAAA,KACK,IAAIH,QAAQ,CAACI,MAAM,KAAK,IAAI,EAAE;UAC/B;UACA;QACJ;QACA;AAChB;AACA;QACgB;QAAA,KACK,IAAI,CAACJ,QAAQ,CAACK,KAAK,EAAE;UACtBf,OAAO,CAACC,KAAK,CACT,2DAA2DL,MAAM,CAACM,OAAO,IAC7E,CAAC;UACDF,OAAO,CAACgB,GAAG,CAAC;YAAEN;UAAS,CAAC,CAAC;UACzB;QACJ;QACA;AAChB;AACA;QACgB,MAAMO,IAAI,GAAG,MAAM,IAAI,CAACxB,UAAU,CAACyB,UAAU,CAACR,QAAQ,CAACO,IAAI,CAAC;QAC5D;AAChB;AACA;AACA;AACA;AACA;AACA;QACgB,IAAIA,IAAI,KAAKE,SAAS,IAAIF,IAAI,KAAK,IAAI,EAAE;UACrCjB,OAAO,CAACC,KAAK,CACT,2DAA2DL,MAAM,CAACG,SAAS,SAASQ,EAAE,eAC1F,CAAC;UACD;QACJ;QAEAZ,UAAU,CAACyB,MAAM,CAAC;UACdb,EAAE;UACFQ,KAAK,EAAEL,QAAQ,CAACK,KAAK;UACrBE;QACJ,CAAC,CAAC;MACN,CAAC,MAAM,IAAIrB,MAAM,CAACG,SAAS,KAAKX,aAAa,CAACiC,MAAM,EAAE;QAClD,MAAMC,QAAQ,GAAGjC,UAAU,CAAsBO,MAAM,CAACE,QAAQ,CAACyB,QAAQ,CAAC;QAC1E;AAChB;AACA;QACgB,IAAI,CAACD,QAAQ,EAAEP,KAAK,EAAE;UAClB;QACJ;QACApB,UAAU,CAAC6B,MAAM,CAAC;UACdjB,EAAE;UACFQ,KAAK,EAAEO,QAAQ,CAACP;QACpB,CAAC,CAAC;MACN;IACJ;IACA,OAAOpB,UAAU;EACrB;AACJ","ignoreList":[]}

package/README.md

@@ -1,9 +1,11 @@
 # @webiny/api-dynamodb-to-elasticsearch
-[![](https://img.shields.io/npm/dw/@webiny/api-file-manager.svg)](https://www.npmjs.com/package/@webiny/api-file-manager) 
-[![](https://img.shields.io/npm/v/@webiny/api-file-manager.svg)](https://www.npmjs.com/package/@webiny/api-file-manager)
-[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
-[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
 
-NOTE: not to be used as a standalone package! 
+> [!NOTE]
+> This package is part of the [Webiny](https://www.webiny.com) monorepo.
+> It’s **included in every Webiny project by default** and is not meant to be used as a standalone package.
 
-This package provides a handler plugin for Webiny, which takes records from DynamoDB Stream and synchronizes them with the Elasticsearch domain configured for current request context. 
+📘 **Documentation:** [https://www.webiny.com/docs](https://www.webiny.com/docs)
+
+---
+
+_This README file is automatically generated during the publish process._

package/SynchronizationBuilder.d.ts

@@ -0,0 +1,24 @@
+import type { Context, IDeleteOperationParams, IInsertOperationParams, IModifyOperationParams } from "./types.js";
+import type { IExecuteWithRetryParams } from "./executeWithRetry.js";
+import type { ITimer } from "@webiny/handler-aws";
+export type ISynchronizationBuilderExecuteWithRetryParams = Omit<IExecuteWithRetryParams, "context" | "timer" | "maxRunningTime" | "operations">;
+export interface ISynchronizationBuilder {
+    insert(params: IInsertOperationParams): void;
+    delete(params: IDeleteOperationParams): void;
+    build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;
+}
+export interface ISynchronizationBuilderParams {
+    timer: ITimer;
+    context: Pick<Context, "elasticsearch">;
+}
+export declare class SynchronizationBuilder implements ISynchronizationBuilder {
+    private readonly timer;
+    private readonly context;
+    private readonly operations;
+    constructor(params: ISynchronizationBuilderParams);
+    insert(params: IInsertOperationParams): void;
+    modify(params: IModifyOperationParams): void;
+    delete(params: IDeleteOperationParams): void;
+    build(): (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;
+}
+export declare const createSynchronizationBuilder: (params: ISynchronizationBuilderParams) => ISynchronizationBuilder;

package/SynchronizationBuilder.js

@@ -0,0 +1,38 @@
+import { Operations } from "./Operations.js";
+import { executeWithRetry } from "./executeWithRetry.js";
+export class SynchronizationBuilder {
+  constructor(params) {
+    this.timer = params.timer;
+    this.context = params.context;
+    this.operations = new Operations();
+  }
+  insert(params) {
+    return this.operations.insert(params);
+  }
+  modify(params) {
+    return this.operations.modify(params);
+  }
+  delete(params) {
+    return this.operations.delete(params);
+  }
+  build() {
+    return async params => {
+      if (this.operations.total === 0) {
+        return;
+      }
+      await executeWithRetry({
+        ...params,
+        maxRunningTime: this.timer.getRemainingMilliseconds(),
+        timer: this.timer,
+        context: this.context,
+        operations: this.operations
+      });
+      this.operations.clear();
+    };
+  }
+}
+export const createSynchronizationBuilder = params => {
+  return new SynchronizationBuilder(params);
+};
+
+//# sourceMappingURL=SynchronizationBuilder.js.map

package/SynchronizationBuilder.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["Operations","executeWithRetry","SynchronizationBuilder","constructor","params","timer","context","operations","insert","modify","delete","build","total","maxRunningTime","getRemainingMilliseconds","clear","createSynchronizationBuilder"],"sources":["SynchronizationBuilder.ts"],"sourcesContent":["import type {\n    Context,\n    IDeleteOperationParams,\n    IInsertOperationParams,\n    IModifyOperationParams,\n    IOperations\n} from \"~/types.js\";\nimport { Operations } from \"~/Operations.js\";\nimport type { IExecuteWithRetryParams } from \"~/executeWithRetry.js\";\nimport { executeWithRetry } from \"~/executeWithRetry.js\";\nimport type { ITimer } from \"@webiny/handler-aws\";\n\nexport type ISynchronizationBuilderExecuteWithRetryParams = Omit<\n    IExecuteWithRetryParams,\n    \"context\" | \"timer\" | \"maxRunningTime\" | \"operations\"\n>;\n\nexport interface ISynchronizationBuilder {\n    insert(params: IInsertOperationParams): void;\n    delete(params: IDeleteOperationParams): void;\n    build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;\n}\n\nexport interface ISynchronizationBuilderParams {\n    timer: ITimer;\n    context: Pick<Context, \"elasticsearch\">;\n}\n\nexport class SynchronizationBuilder implements ISynchronizationBuilder {\n    private readonly timer: ITimer;\n    private readonly context: Pick<Context, \"elasticsearch\">;\n    private readonly operations: IOperations;\n\n    public constructor(params: ISynchronizationBuilderParams) {\n        this.timer = params.timer;\n        this.context = params.context;\n        this.operations = new Operations();\n    }\n\n    public insert(params: IInsertOperationParams): void {\n        return this.operations.insert(params);\n    }\n\n    public modify(params: IModifyOperationParams): void {\n        return this.operations.modify(params);\n    }\n\n    public delete(params: IDeleteOperationParams): void {\n        return this.operations.delete(params);\n    }\n\n    public build() {\n        return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => {\n            if (this.operations.total === 0) {\n                return;\n            }\n            await executeWithRetry({\n                ...params,\n                maxRunningTime: this.timer.getRemainingMilliseconds(),\n                timer: this.timer,\n                context: this.context,\n                operations: this.operations\n            });\n            this.operations.clear();\n        };\n    }\n}\n\nexport const createSynchronizationBuilder = (\n    params: ISynchronizationBuilderParams\n): ISynchronizationBuilder => {\n    return new SynchronizationBuilder(params);\n};\n"],"mappings":"AAOA,SAASA,UAAU;AAEnB,SAASC,gBAAgB;AAmBzB,OAAO,MAAMC,sBAAsB,CAAoC;EAK5DC,WAAWA,CAACC,MAAqC,EAAE;IACtD,IAAI,CAACC,KAAK,GAAGD,MAAM,CAACC,KAAK;IACzB,IAAI,CAACC,OAAO,GAAGF,MAAM,CAACE,OAAO;IAC7B,IAAI,CAACC,UAAU,GAAG,IAAIP,UAAU,CAAC,CAAC;EACtC;EAEOQ,MAAMA,CAACJ,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACC,MAAM,CAACJ,MAAM,CAAC;EACzC;EAEOK,MAAMA,CAACL,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACE,MAAM,CAACL,MAAM,CAAC;EACzC;EAEOM,MAAMA,CAACN,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACG,MAAM,CAACN,MAAM,CAAC;EACzC;EAEOO,KAAKA,CAAA,EAAG;IACX,OAAO,MAAOP,MAAsD,IAAK;MACrE,IAAI,IAAI,CAACG,UAAU,CAACK,KAAK,KAAK,CAAC,EAAE;QAC7B;MACJ;MACA,MAAMX,gBAAgB,CAAC;QACnB,GAAGG,MAAM;QACTS,cAAc,EAAE,IAAI,CAACR,KAAK,CAACS,wBAAwB,CAAC,CAAC;QACrDT,KAAK,EAAE,IAAI,CAACA,KAAK;QACjBC,OAAO,EAAE,IAAI,CAACA,OAAO;QACrBC,UAAU,EAAE,IAAI,CAACA;MACrB,CAAC,CAAC;MACF,IAAI,CAACA,UAAU,CAACQ,KAAK,CAAC,CAAC;IAC3B,CAAC;EACL;AACJ;AAEA,OAAO,MAAMC,4BAA4B,GACrCZ,MAAqC,IACX;EAC1B,OAAO,IAAIF,sBAAsB,CAACE,MAAM,CAAC;AAC7C,CAAC","ignoreList":[]}

package/eventHandler.d.ts

@@ -0,0 +1 @@
+export declare const createEventHandler: () => import("@webiny/handler-aws").DynamoDBEventHandler<unknown>;

package/eventHandler.js

@@ -0,0 +1,48 @@
+import { createDynamoDBEventHandler, timerFactory } from "@webiny/handler-aws";
+import { OperationsBuilder } from "./OperationsBuilder.js";
+import { executeWithRetry } from "./executeWithRetry.js";
+
+/**
+ * Also, we need to set the maximum running time for the Lambda Function.
+ * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232
+ * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218
+ */
+const MAX_RUNNING_TIME = 900;
+export const createEventHandler = () => {
+  return createDynamoDBEventHandler(async ({
+    event,
+    context: ctx,
+    lambdaContext
+  }) => {
+    const timer = timerFactory(lambdaContext);
+    const context = ctx;
+    if (!context.elasticsearch) {
+      console.error("Missing elasticsearch definition on context.");
+      return null;
+    }
+    const builder = new OperationsBuilder({
+      compressor: context.compressor
+    });
+    const operations = await builder.build({
+      records: event.Records
+    });
+    /**
+     * No need to do anything if there are no operations.
+     */
+    if (operations.total === 0) {
+      return null;
+    }
+    /**
+     * Execute the operations with retry.
+     */
+    await executeWithRetry({
+      timer,
+      maxRunningTime: MAX_RUNNING_TIME,
+      context,
+      operations
+    });
+    return null;
+  });
+};
+
+//# sourceMappingURL=eventHandler.js.map

package/eventHandler.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["createDynamoDBEventHandler","timerFactory","OperationsBuilder","executeWithRetry","MAX_RUNNING_TIME","createEventHandler","event","context","ctx","lambdaContext","timer","elasticsearch","console","error","builder","compressor","operations","build","records","Records","total","maxRunningTime"],"sources":["eventHandler.ts"],"sourcesContent":["import { createDynamoDBEventHandler, timerFactory } from \"@webiny/handler-aws\";\nimport type { Context } from \"~/types.js\";\nimport { OperationsBuilder } from \"~/OperationsBuilder.js\";\nimport { executeWithRetry } from \"~/executeWithRetry.js\";\n\n/**\n * Also, we need to set the maximum running time for the Lambda Function.\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218\n */\nconst MAX_RUNNING_TIME = 900;\n\nexport const createEventHandler = () => {\n    return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => {\n        const timer = timerFactory(lambdaContext);\n        const context = ctx as unknown as Context;\n        if (!context.elasticsearch) {\n            console.error(\"Missing elasticsearch definition on context.\");\n            return null;\n        }\n\n        const builder = new OperationsBuilder({\n            compressor: context.compressor\n        });\n\n        const operations = await builder.build({\n            records: event.Records\n        });\n        /**\n         * No need to do anything if there are no operations.\n         */\n        if (operations.total === 0) {\n            return null;\n        }\n        /**\n         * Execute the operations with retry.\n         */\n        await executeWithRetry({\n            timer,\n            maxRunningTime: MAX_RUNNING_TIME,\n            context,\n            operations\n        });\n\n        return null;\n    });\n};\n"],"mappings":"AAAA,SAASA,0BAA0B,EAAEC,YAAY,QAAQ,qBAAqB;AAE9E,SAASC,iBAAiB;AAC1B,SAASC,gBAAgB;;AAEzB;AACA;AACA;AACA;AACA;AACA,MAAMC,gBAAgB,GAAG,GAAG;AAE5B,OAAO,MAAMC,kBAAkB,GAAGA,CAAA,KAAM;EACpC,OAAOL,0BAA0B,CAAC,OAAO;IAAEM,KAAK;IAAEC,OAAO,EAAEC,GAAG;IAAEC;EAAc,CAAC,KAAK;IAChF,MAAMC,KAAK,GAAGT,YAAY,CAACQ,aAAa,CAAC;IACzC,MAAMF,OAAO,GAAGC,GAAyB;IACzC,IAAI,CAACD,OAAO,CAACI,aAAa,EAAE;MACxBC,OAAO,CAACC,KAAK,CAAC,8CAA8C,CAAC;MAC7D,OAAO,IAAI;IACf;IAEA,MAAMC,OAAO,GAAG,IAAIZ,iBAAiB,CAAC;MAClCa,UAAU,EAAER,OAAO,CAACQ;IACxB,CAAC,CAAC;IAEF,MAAMC,UAAU,GAAG,MAAMF,OAAO,CAACG,KAAK,CAAC;MACnCC,OAAO,EAAEZ,KAAK,CAACa;IACnB,CAAC,CAAC;IACF;AACR;AACA;IACQ,IAAIH,UAAU,CAACI,KAAK,KAAK,CAAC,EAAE;MACxB,OAAO,IAAI;IACf;IACA;AACR;AACA;IACQ,MAAMjB,gBAAgB,CAAC;MACnBO,KAAK;MACLW,cAAc,EAAEjB,gBAAgB;MAChCG,OAAO;MACPS;IACJ,CAAC,CAAC;IAEF,OAAO,IAAI;EACf,CAAC,CAAC;AACN,CAAC","ignoreList":[]}

package/execute.d.ts

@@ -0,0 +1,23 @@
+import type { ITimer } from "@webiny/handler-aws";
+import type { Context, IOperations } from "./types.js";
+export interface BulkOperationsResponseBodyItemIndexError {
+    reason?: string;
+}
+export interface BulkOperationsResponseBodyItemIndex {
+    error?: BulkOperationsResponseBodyItemIndexError;
+}
+export interface BulkOperationsResponseBodyItem {
+    index?: BulkOperationsResponseBodyItemIndex;
+    error?: string;
+}
+export interface BulkOperationsResponseBody {
+    items: BulkOperationsResponseBodyItem[];
+}
+export interface IExecuteParams {
+    timer: ITimer;
+    maxRunningTime: number;
+    maxProcessorPercent: number;
+    context: Pick<Context, "elasticsearch">;
+    operations: Pick<IOperations, "items" | "total">;
+}
+export declare const execute: (params: IExecuteParams) => () => Promise<void>;

package/execute.js

@@ -0,0 +1,137 @@
+import { createWaitUntilHealthy, ElasticsearchCatClusterHealthStatus, UnhealthyClusterError, WaitingHealthyClusterAbortedError } from "@webiny/api-elasticsearch";
+import { WebinyError } from "@webiny/error";
+import { shouldShowLogs } from "./helpers/shouldShowLogs.js";
+const getError = item => {
+  if (!item.index?.error?.reason) {
+    return null;
+  }
+  const reason = item.index.error.reason;
+  if (reason.match(/no such index \[([a-zA-Z0-9_-]+)\]/) !== null) {
+    return "index";
+  }
+  return reason;
+};
+const checkErrors = result => {
+  if (!result || !result.body || !result.body.items) {
+    return;
+  }
+  for (const item of result.body.items) {
+    const err = getError(item);
+    if (!err) {
+      continue;
+    } else if (err === "index") {
+      if (process.env.DEBUG === "true") {
+        console.error("Bulk response", JSON.stringify(result, null, 2));
+      }
+      continue;
+    }
+    console.error("Body item with error", item);
+    throw new WebinyError(err, "DYNAMODB_TO_OPENSEARCH_ERROR", item);
+  }
+};
+export const execute = params => {
+  return async () => {
+    const {
+      context,
+      timer,
+      maxRunningTime,
+      maxProcessorPercent,
+      operations
+    } = params;
+    if (operations.total === 0) {
+      return;
+    }
+    const remainingTime = timer.getRemainingSeconds();
+    const runningTime = maxRunningTime - remainingTime;
+    const maxWaitingTime = remainingTime - 90;
+    if (shouldShowLogs()) {
+      console.debug(`The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`);
+    }
+    const healthCheck = createWaitUntilHealthy(context.elasticsearch, {
+      minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,
+      waitingTimeStep: 30,
+      maxProcessorPercent,
+      maxWaitingTime
+    });
+
+    // const log = context.logger.withSource("dynamodbToElasticsearch");
+    const log = {
+      notice: (...params) => {
+        console.log(...params);
+      },
+      debug: (...params) => {
+        console.debug(...params);
+      },
+      info: (...params) => {
+        console.info(...params);
+      },
+      warn: (...params) => {
+        console.warn(...params);
+      },
+      error: (...params) => {
+        console.error(...params);
+      }
+    };
+    try {
+      await healthCheck.wait({
+        async onUnhealthy({
+          startedAt,
+          runs,
+          mustEndAt,
+          waitingTimeStep,
+          waitingReason
+        }) {
+          console.debug(`Cluster is unhealthy on run #${runs}.`, {
+            startedAt,
+            mustEndAt,
+            waitingTimeStep,
+            waitingReason
+          });
+        },
+        async onTimeout({
+          startedAt,
+          runs,
+          waitingTimeStep,
+          mustEndAt,
+          waitingReason
+        }) {
+          console.error(`Cluster health check timeout on run #${runs}.`, {
+            startedAt,
+            mustEndAt,
+            waitingTimeStep,
+            waitingReason
+          });
+        }
+      });
+    } catch (ex) {
+      if (ex instanceof UnhealthyClusterError || ex instanceof WaitingHealthyClusterAbortedError) {
+        throw ex;
+      }
+      console.error(`Cluster health check failed.`, ex);
+      throw ex;
+    }
+    try {
+      const res = await context.elasticsearch.bulk({
+        body: operations.items
+      });
+      checkErrors(res);
+    } catch (error) {
+      log.error(error, {
+        tenant: "root"
+      });
+      if (shouldShowLogs() === false) {
+        throw error;
+      }
+      const meta = error?.meta || {};
+      delete meta["meta"];
+      console.error("Bulk error", JSON.stringify(error, null, 2));
+      throw error;
+    }
+    if (shouldShowLogs() === false) {
+      return;
+    }
+    console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);
+  };
+};
+
+//# sourceMappingURL=execute.js.map

package/execute.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["createWaitUntilHealthy","ElasticsearchCatClusterHealthStatus","UnhealthyClusterError","WaitingHealthyClusterAbortedError","WebinyError","shouldShowLogs","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","total","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","elasticsearch","minClusterHealthStatus","Yellow","waitingTimeStep","log","notice","info","warn","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","res","bulk","tenant","meta"],"sources":["execute.ts"],"sourcesContent":["import {\n    createWaitUntilHealthy,\n    ElasticsearchCatClusterHealthStatus,\n    UnhealthyClusterError,\n    WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport type { ApiResponse } from \"@webiny/api-elasticsearch/types.js\";\nimport { WebinyError } from \"@webiny/error\";\nimport type { Context, IOperations } from \"./types.js\";\nimport { shouldShowLogs } from \"~/helpers/shouldShowLogs.js\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n    reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n    error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n    index?: BulkOperationsResponseBodyItemIndex;\n    error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n    items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n    timer: ITimer;\n    maxRunningTime: number;\n    maxProcessorPercent: number;\n    context: Pick<Context, \"elasticsearch\">;\n    operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n    if (!item.index?.error?.reason) {\n        return null;\n    }\n    const reason = item.index.error.reason;\n    if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n        return \"index\";\n    }\n    return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n    if (!result || !result.body || !result.body.items) {\n        return;\n    }\n    for (const item of result.body.items) {\n        const err = getError(item);\n        if (!err) {\n            continue;\n        } else if (err === \"index\") {\n            if (process.env.DEBUG === \"true\") {\n                console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n            }\n            continue;\n        }\n        console.error(\"Body item with error\", item);\n        throw new WebinyError(err, \"DYNAMODB_TO_OPENSEARCH_ERROR\", item);\n    }\n};\n\nexport const execute = (params: IExecuteParams) => {\n    return async (): Promise<void> => {\n        const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n        if (operations.total === 0) {\n            return;\n        }\n\n        const remainingTime = timer.getRemainingSeconds();\n        const runningTime = maxRunningTime - remainingTime;\n        const maxWaitingTime = remainingTime - 90;\n\n        if (shouldShowLogs()) {\n            console.debug(\n                `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n            );\n        }\n\n        const healthCheck = createWaitUntilHealthy(context.elasticsearch, {\n            minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,\n            waitingTimeStep: 30,\n            maxProcessorPercent,\n            maxWaitingTime\n        });\n\n        // const log = context.logger.withSource(\"dynamodbToElasticsearch\");\n        const log = {\n            notice: (...params: any[]) => {\n                console.log(...params);\n            },\n            debug: (...params: any[]) => {\n                console.debug(...params);\n            },\n            info: (...params: any[]) => {\n                console.info(...params);\n            },\n            warn: (...params: any[]) => {\n                console.warn(...params);\n            },\n            error: (...params: any[]) => {\n                console.error(...params);\n            }\n        };\n\n        try {\n            await healthCheck.wait({\n                async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n                    console.debug(`Cluster is unhealthy on run #${runs}.`, {\n                        startedAt,\n                        mustEndAt,\n                        waitingTimeStep,\n                        waitingReason\n                    });\n                },\n                async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n                    console.error(`Cluster health check timeout on run #${runs}.`, {\n                        startedAt,\n                        mustEndAt,\n                        waitingTimeStep,\n                        waitingReason\n                    });\n                }\n            });\n        } catch (ex) {\n            if (\n                ex instanceof UnhealthyClusterError ||\n                ex instanceof WaitingHealthyClusterAbortedError\n            ) {\n                throw ex;\n            }\n            console.error(`Cluster health check failed.`, ex);\n            throw ex;\n        }\n\n        try {\n            const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n                body: operations.items\n            });\n            checkErrors(res);\n        } catch (error) {\n            log.error(error, {\n                tenant: \"root\"\n            });\n\n            if (shouldShowLogs() === false) {\n                throw error;\n            }\n            const meta = error?.meta || {};\n            delete meta[\"meta\"];\n            console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n            throw error;\n        }\n        if (shouldShowLogs() === false) {\n            return;\n        }\n        console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n    };\n};\n"],"mappings":"AAAA,SACIA,sBAAsB,EACtBC,mCAAmC,EACnCC,qBAAqB,EACrBC,iCAAiC,QAC9B,2BAA2B;AAGlC,SAASC,WAAW,QAAQ,eAAe;AAE3C,SAASC,cAAc;AA2BvB,MAAMC,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,EAAEC,KAAK,EAAEC,MAAM,EAAE;IAC5B,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMR,IAAI,IAAIM,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACS,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACX,KAAK,CAAC,eAAe,EAAEY,IAAI,CAACC,SAAS,CAACT,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAO,OAAO,CAACX,KAAK,CAAC,sBAAsB,EAAEF,IAAI,CAAC;IAC3C,MAAM,IAAIH,WAAW,CAACY,GAAG,EAAE,8BAA8B,EAAET,IAAI,CAAC;EACpE;AACJ,CAAC;AAED,OAAO,MAAMgB,OAAO,GAAIC,MAAsB,IAAK;EAC/C,OAAO,YAA2B;IAC9B,MAAM;MAAEC,OAAO;MAAEC,KAAK;MAAEC,cAAc;MAAEC,mBAAmB;MAAEC;IAAW,CAAC,GAAGL,MAAM;IAElF,IAAIK,UAAU,CAACC,KAAK,KAAK,CAAC,EAAE;MACxB;IACJ;IAEA,MAAMC,aAAa,GAAGL,KAAK,CAACM,mBAAmB,CAAC,CAAC;IACjD,MAAMC,WAAW,GAAGN,cAAc,GAAGI,aAAa;IAClD,MAAMG,cAAc,GAAGH,aAAa,GAAG,EAAE;IAEzC,IAAI1B,cAAc,CAAC,CAAC,EAAE;MAClBe,OAAO,CAACe,KAAK,CACT,qCAAqCF,WAAW,6CAA6CC,cAAc,GAC/G,CAAC;IACL;IAEA,MAAME,WAAW,GAAGpC,sBAAsB,CAACyB,OAAO,CAACY,aAAa,EAAE;MAC9DC,sBAAsB,EAAErC,mCAAmC,CAACsC,MAAM;MAClEC,eAAe,EAAE,EAAE;MACnBZ,mBAAmB;MACnBM;IACJ,CAAC,CAAC;;IAEF;IACA,MAAMO,GAAG,GAAG;MACRC,MAAM,EAAEA,CAAC,GAAGlB,MAAa,KAAK;QAC1BJ,OAAO,CAACqB,GAAG,CAAC,GAAGjB,MAAM,CAAC;MAC1B,CAAC;MACDW,KAAK,EAAEA,CAAC,GAAGX,MAAa,KAAK;QACzBJ,OAAO,CAACe,KAAK,CAAC,GAAGX,MAAM,CAAC;MAC5B,CAAC;MACDmB,IAAI,EAAEA,CAAC,GAAGnB,MAAa,KAAK;QACxBJ,OAAO,CAACuB,IAAI,CAAC,GAAGnB,MAAM,CAAC;MAC3B,CAAC;MACDoB,IAAI,EAAEA,CAAC,GAAGpB,MAAa,KAAK;QACxBJ,OAAO,CAACwB,IAAI,CAAC,GAAGpB,MAAM,CAAC;MAC3B,CAAC;MACDf,KAAK,EAAEA,CAAC,GAAGe,MAAa,KAAK;QACzBJ,OAAO,CAACX,KAAK,CAAC,GAAGe,MAAM,CAAC;MAC5B;IACJ,CAAC;IAED,IAAI;MACA,MAAMY,WAAW,CAACS,IAAI,CAAC;QACnB,MAAMC,WAAWA,CAAC;UAAEC,SAAS;UAAEC,IAAI;UAAEC,SAAS;UAAET,eAAe;UAAEU;QAAc,CAAC,EAAE;UAC9E9B,OAAO,CAACe,KAAK,CAAC,gCAAgCa,IAAI,GAAG,EAAE;YACnDD,SAAS;YACTE,SAAS;YACTT,eAAe;YACfU;UACJ,CAAC,CAAC;QACN,CAAC;QACD,MAAMC,SAASA,CAAC;UAAEJ,SAAS;UAAEC,IAAI;UAAER,eAAe;UAAES,SAAS;UAAEC;QAAc,CAAC,EAAE;UAC5E9B,OAAO,CAACX,KAAK,CAAC,wCAAwCuC,IAAI,GAAG,EAAE;YAC3DD,SAAS;YACTE,SAAS;YACTT,eAAe;YACfU;UACJ,CAAC,CAAC;QACN;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,IACIA,EAAE,YAAYlD,qBAAqB,IACnCkD,EAAE,YAAYjD,iCAAiC,EACjD;QACE,MAAMiD,EAAE;MACZ;MACAhC,OAAO,CAACX,KAAK,CAAC,8BAA8B,EAAE2C,EAAE,CAAC;MACjD,MAAMA,EAAE;IACZ;IAEA,IAAI;MACA,MAAMC,GAAG,GAAG,MAAM5B,OAAO,CAACY,aAAa,CAACiB,IAAI,CAA6B;QACrExC,IAAI,EAAEe,UAAU,CAACd;MACrB,CAAC,CAAC;MACFH,WAAW,CAACyC,GAAG,CAAC;IACpB,CAAC,CAAC,OAAO5C,KAAK,EAAE;MACZgC,GAAG,CAAChC,KAAK,CAACA,KAAK,EAAE;QACb8C,MAAM,EAAE;MACZ,CAAC,CAAC;MAEF,IAAIlD,cAAc,CAAC,CAAC,KAAK,KAAK,EAAE;QAC5B,MAAMI,KAAK;MACf;MACA,MAAM+C,IAAI,GAAG/C,KAAK,EAAE+C,IAAI,IAAI,CAAC,CAAC;MAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;MACnBpC,OAAO,CAACX,KAAK,CAAC,YAAY,EAAEY,IAAI,CAACC,SAAS,CAACb,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MAC3D,MAAMA,KAAK;IACf;IACA,IAAIJ,cAAc,CAAC,CAAC,KAAK,KAAK,EAAE;MAC5B;IACJ;IACAe,OAAO,CAACuB,IAAI,CAAC,eAAed,UAAU,CAACC,KAAK,sCAAsC,CAAC;EACvF,CAAC;AACL,CAAC","ignoreList":[]}

package/executeWithRetry.d.ts

@@ -0,0 +1,9 @@
+import type { IExecuteParams } from "./execute.js";
+export interface IExecuteWithRetryParams extends Omit<IExecuteParams, "maxProcessorPercent"> {
+    maxRetryTime?: number;
+    retries?: number;
+    minTimeout?: number;
+    maxTimeout?: number;
+    maxProcessorPercent?: number;
+}
+export declare const executeWithRetry: (params: IExecuteWithRetryParams) => Promise<void>;

package/executeWithRetry.js

@@ -0,0 +1,47 @@
+import { execute } from "./execute.js";
+import { NotEnoughRemainingTimeError } from "./NotEnoughRemainingTimeError.js";
+import pRetry from "p-retry";
+import { getNumberEnvVariable } from "./helpers/getNumberEnvVariable.js";
+const minRemainingSecondsToTimeout = 120;
+const MAX_PROCESSOR_PERCENT = getNumberEnvVariable("MAX_ES_PROCESSOR", process.env.NODE_ENV === "test" ? 101 : 98);
+export const executeWithRetry = async params => {
+  const maxRetryTime = getNumberEnvVariable("WEBINY_DYNAMODB_TO_OPENSEARCH_MAX_RETRY_TIME", params.maxRetryTime || 300000);
+  const retries = getNumberEnvVariable("WEBINY_DYNAMODB_TO_OPENSEARCH_RETRIES", params.retries || 20);
+  const minTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_OPENSEARCH_MIN_TIMEOUT", params.minTimeout || 1500);
+  const maxTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_OPENSEARCH_MAX_TIMEOUT", params.maxTimeout || 30000);
+  try {
+    await pRetry(execute({
+      timer: params.timer,
+      maxRunningTime: params.maxRunningTime,
+      maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,
+      context: params.context,
+      operations: params.operations
+    }), {
+      maxRetryTime,
+      retries,
+      minTimeout,
+      maxTimeout,
+      onFailedAttempt: ({
+        error,
+        attemptNumber
+      }) => {
+        if (params.timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {
+          throw new NotEnoughRemainingTimeError(error);
+        }
+        /**
+         * We will only log attempts which are after 3/4 of total attempts.
+         */
+        if (attemptNumber < retries * 0.75) {
+          return;
+        }
+        console.error(`Attempt #${attemptNumber} failed.`);
+        console.error(error);
+      }
+    });
+  } catch (ex) {
+    // TODO implement storing of failed operations
+    throw ex;
+  }
+};
+
+//# sourceMappingURL=executeWithRetry.js.map

package/executeWithRetry.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["execute","NotEnoughRemainingTimeError","pRetry","getNumberEnvVariable","minRemainingSecondsToTimeout","MAX_PROCESSOR_PERCENT","process","env","NODE_ENV","executeWithRetry","params","maxRetryTime","retries","minTimeout","maxTimeout","timer","maxRunningTime","maxProcessorPercent","context","operations","onFailedAttempt","error","attemptNumber","getRemainingSeconds","console","ex"],"sources":["executeWithRetry.ts"],"sourcesContent":["import type { IExecuteParams } from \"~/execute.js\";\nimport { execute } from \"~/execute.js\";\nimport { NotEnoughRemainingTimeError } from \"~/NotEnoughRemainingTimeError.js\";\nimport pRetry from \"p-retry\";\nimport { getNumberEnvVariable } from \"./helpers/getNumberEnvVariable.js\";\n\nconst minRemainingSecondsToTimeout = 120;\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n    \"MAX_ES_PROCESSOR\",\n    process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\nexport interface IExecuteWithRetryParams extends Omit<IExecuteParams, \"maxProcessorPercent\"> {\n    maxRetryTime?: number;\n    retries?: number;\n    minTimeout?: number;\n    maxTimeout?: number;\n    maxProcessorPercent?: number;\n}\n\nexport const executeWithRetry = async (params: IExecuteWithRetryParams) => {\n    const maxRetryTime = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_OPENSEARCH_MAX_RETRY_TIME\",\n        params.maxRetryTime || 300000\n    );\n    const retries = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_OPENSEARCH_RETRIES\",\n        params.retries || 20\n    );\n    const minTimeout = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_OPENSEARCH_MIN_TIMEOUT\",\n        params.minTimeout || 1500\n    );\n    const maxTimeout = getNumberEnvVariable(\n        \"WEBINY_DYNAMODB_TO_OPENSEARCH_MAX_TIMEOUT\",\n        params.maxTimeout || 30000\n    );\n\n    try {\n        await pRetry(\n            execute({\n                timer: params.timer,\n                maxRunningTime: params.maxRunningTime,\n                maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,\n                context: params.context,\n                operations: params.operations\n            }),\n            {\n                maxRetryTime,\n                retries,\n                minTimeout,\n                maxTimeout,\n                onFailedAttempt: ({ error, attemptNumber }) => {\n                    if (params.timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {\n                        throw new NotEnoughRemainingTimeError(error);\n                    }\n                    /**\n                     * We will only log attempts which are after 3/4 of total attempts.\n                     */\n                    if (attemptNumber < retries * 0.75) {\n                        return;\n                    }\n                    console.error(`Attempt #${attemptNumber} failed.`);\n                    console.error(error);\n                }\n            }\n        );\n    } catch (ex) {\n        // TODO implement storing of failed operations\n        throw ex;\n    }\n};\n"],"mappings":"AACA,SAASA,OAAO;AAChB,SAASC,2BAA2B;AACpC,OAAOC,MAAM,MAAM,SAAS;AAC5B,SAASC,oBAAoB;AAE7B,MAAMC,4BAA4B,GAAG,GAAG;AAExC,MAAMC,qBAAqB,GAAGF,oBAAoB,CAC9C,kBAAkB,EAClBG,OAAO,CAACC,GAAG,CAACC,QAAQ,KAAK,MAAM,GAAG,GAAG,GAAG,EAC5C,CAAC;AAUD,OAAO,MAAMC,gBAAgB,GAAG,MAAOC,MAA+B,IAAK;EACvE,MAAMC,YAAY,GAAGR,oBAAoB,CACrC,8CAA8C,EAC9CO,MAAM,CAACC,YAAY,IAAI,MAC3B,CAAC;EACD,MAAMC,OAAO,GAAGT,oBAAoB,CAChC,uCAAuC,EACvCO,MAAM,CAACE,OAAO,IAAI,EACtB,CAAC;EACD,MAAMC,UAAU,GAAGV,oBAAoB,CACnC,2CAA2C,EAC3CO,MAAM,CAACG,UAAU,IAAI,IACzB,CAAC;EACD,MAAMC,UAAU,GAAGX,oBAAoB,CACnC,2CAA2C,EAC3CO,MAAM,CAACI,UAAU,IAAI,KACzB,CAAC;EAED,IAAI;IACA,MAAMZ,MAAM,CACRF,OAAO,CAAC;MACJe,KAAK,EAAEL,MAAM,CAACK,KAAK;MACnBC,cAAc,EAAEN,MAAM,CAACM,cAAc;MACrCC,mBAAmB,EAAEP,MAAM,CAACO,mBAAmB,IAAIZ,qBAAqB;MACxEa,OAAO,EAAER,MAAM,CAACQ,OAAO;MACvBC,UAAU,EAAET,MAAM,CAACS;IACvB,CAAC,CAAC,EACF;MACIR,YAAY;MACZC,OAAO;MACPC,UAAU;MACVC,UAAU;MACVM,eAAe,EAAEA,CAAC;QAAEC,KAAK;QAAEC;MAAc,CAAC,KAAK;QAC3C,IAAIZ,MAAM,CAACK,KAAK,CAACQ,mBAAmB,CAAC,CAAC,GAAGnB,4BAA4B,EAAE;UACnE,MAAM,IAAIH,2BAA2B,CAACoB,KAAK,CAAC;QAChD;QACA;AACpB;AACA;QACoB,IAAIC,aAAa,GAAGV,OAAO,GAAG,IAAI,EAAE;UAChC;QACJ;QACAY,OAAO,CAACH,KAAK,CAAC,YAAYC,aAAa,UAAU,CAAC;QAClDE,OAAO,CAACH,KAAK,CAACA,KAAK,CAAC;MACxB;IACJ,CACJ,CAAC;EACL,CAAC,CAAC,OAAOI,EAAE,EAAE;IACT;IACA,MAAMA,EAAE;EACZ;AACJ,CAAC","ignoreList":[]}

package/helpers/getNumberEnvVariable.d.ts

@@ -0,0 +1 @@
+export declare const getNumberEnvVariable: (name: string, def: number) => number;

package/helpers/getNumberEnvVariable.js

@@ -0,0 +1,12 @@
+export const getNumberEnvVariable = (name, def) => {
+  const input = process.env[name];
+  const value = Number(input);
+  if (isNaN(value)) {
+    return def;
+  } else if (value <= 0) {
+    return def;
+  }
+  return value;
+};
+
+//# sourceMappingURL=getNumberEnvVariable.js.map

package/helpers/getNumberEnvVariable.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["getNumberEnvVariable","name","def","input","process","env","value","Number","isNaN"],"sources":["getNumberEnvVariable.ts"],"sourcesContent":["export const getNumberEnvVariable = (name: string, def: number): number => {\n    const input = process.env[name];\n    const value = Number(input);\n    if (isNaN(value)) {\n        return def;\n    } else if (value <= 0) {\n        return def;\n    }\n    return value;\n};\n"],"mappings":"AAAA,OAAO,MAAMA,oBAAoB,GAAGA,CAACC,IAAY,EAAEC,GAAW,KAAa;EACvE,MAAMC,KAAK,GAAGC,OAAO,CAACC,GAAG,CAACJ,IAAI,CAAC;EAC/B,MAAMK,KAAK,GAAGC,MAAM,CAACJ,KAAK,CAAC;EAC3B,IAAIK,KAAK,CAACF,KAAK,CAAC,EAAE;IACd,OAAOJ,GAAG;EACd,CAAC,MAAM,IAAII,KAAK,IAAI,CAAC,EAAE;IACnB,OAAOJ,GAAG;EACd;EACA,OAAOI,KAAK;AAChB,CAAC","ignoreList":[]}

package/helpers/shouldShowLogs.d.ts

@@ -0,0 +1 @@
+export declare const shouldShowLogs: () => boolean;

package/helpers/shouldShowLogs.js

@@ -0,0 +1,11 @@
+export const shouldShowLogs = () => {
+  /**
+   * Don't show logs during tests, really no point.
+   */
+  if (process.env.TESTING === "true") {
+    return false;
+  }
+  return process.env.DEBUG === "true";
+};
+
+//# sourceMappingURL=shouldShowLogs.js.map

package/helpers/shouldShowLogs.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["shouldShowLogs","process","env","TESTING","DEBUG"],"sources":["shouldShowLogs.ts"],"sourcesContent":["export const shouldShowLogs = (): boolean => {\n    /**\n     * Don't show logs during tests, really no point.\n     */\n    if (process.env.TESTING === \"true\") {\n        return false;\n    }\n    return process.env.DEBUG === \"true\";\n};\n"],"mappings":"AAAA,OAAO,MAAMA,cAAc,GAAGA,CAAA,KAAe;EACzC;AACJ;AACA;EACI,IAAIC,OAAO,CAACC,GAAG,CAACC,OAAO,KAAK,MAAM,EAAE;IAChC,OAAO,KAAK;EAChB;EACA,OAAOF,OAAO,CAACC,GAAG,CAACE,KAAK,KAAK,MAAM;AACvC,CAAC","ignoreList":[]}

package/index.d.ts

@@ -1,6 +1,9 @@
-export declare enum Operations {
-    INSERT = "INSERT",
-    MODIFY = "MODIFY",
-    REMOVE = "REMOVE"
-}
-export declare const createEventHandler: () => import("@webiny/handler-aws").DynamoDBEventHandler<unknown>;
+export * from "./eventHandler.js";
+export * from "./execute.js";
+export * from "./executeWithRetry.js";
+export * from "./marshall.js";
+export * from "./NotEnoughRemainingTimeError.js";
+export * from "./Operations.js";
+export * from "./OperationsBuilder.js";
+export * from "./SynchronizationBuilder.js";
+export type * from "./types.js";

package/index.js

@@ -1,280 +1,10 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault").default;
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.createEventHandler = exports.Operations = void 0;
-var _error = _interopRequireDefault(require("@webiny/error"));
-var _clientDynamodb = require("@webiny/aws-sdk/client-dynamodb");
-var _apiElasticsearch = require("@webiny/api-elasticsearch");
-var _handlerAws = require("@webiny/handler-aws");
-var _types = require("@webiny/api-elasticsearch/operations/types");
-var _pRetry = _interopRequireDefault(require("p-retry"));
-var _NotEnoughRemainingTimeError = require("./NotEnoughRemainingTimeError");
-let Operations = exports.Operations = /*#__PURE__*/function (Operations) {
-  Operations["INSERT"] = "INSERT";
-  Operations["MODIFY"] = "MODIFY";
-  Operations["REMOVE"] = "REMOVE";
-  return Operations;
-}({});
-const getError = item => {
-  if (!item.index?.error?.reason) {
-    return null;
-  }
-  const reason = item.index.error.reason;
-  if (reason.match(/no such index \[([a-zA-Z0-9_-]+)\]/) !== null) {
-    return "index";
-  }
-  return reason;
-};
-const checkErrors = result => {
-  if (!result || !result.body || !result.body.items) {
-    return;
-  }
-  for (const item of result.body.items) {
-    const err = getError(item);
-    if (!err) {
-      continue;
-    } else if (err === "index") {
-      if (process.env.DEBUG === "true") {
-        console.error("Bulk response", JSON.stringify(result, null, 2));
-      }
-      continue;
-    }
-    console.error(item.error);
-    throw new _error.default(err, "DYNAMODB_TO_ELASTICSEARCH_ERROR", item);
-  }
-};
-const getNumberEnvVariable = (name, def) => {
-  const input = process.env[name];
-  const value = Number(input);
-  if (isNaN(value)) {
-    return def;
-  } else if (value <= 0) {
-    return def;
-  }
-  return value;
-};
-const MAX_PROCESSOR_PERCENT = getNumberEnvVariable("MAX_ES_PROCESSOR", process.env.NODE_ENV === "test" ? 101 : 98);
-const unmarshall = value => {
-  if (!value) {
-    return undefined;
-  }
-  return (0, _clientDynamodb.unmarshall)(value);
-};
-const minRemainingSecondsToTimeout = 120;
-
-/**
- * Also, we need to set the maximum running time for the Lambda Function.
- * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232
- * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218
- */
-const MAX_RUNNING_TIME = 900;
-const createEventHandler = () => {
-  return (0, _handlerAws.createDynamoDBEventHandler)(async ({
-    event,
-    context: ctx,
-    lambdaContext
-  }) => {
-    const timer = (0, _handlerAws.timerFactory)(lambdaContext);
-    const context = ctx;
-    if (!context.elasticsearch) {
-      console.error("Missing elasticsearch definition on context.");
-      return null;
-    }
-    const operations = [];
-    const operationIdList = [];
-    for (const record of event.Records) {
-      const dynamodb = record.dynamodb;
-      if (!dynamodb) {
-        continue;
-      }
-      /**
-       * TODO: figure out correct types
-       */
-      // @ts-expect-error
-      const newImage = unmarshall(dynamodb.NewImage);
-
-      // Note that with the `REMOVE` event, there is no `NewImage` property. Which means,
-      // if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still
-      // need to process it.
-      if (newImage && newImage.ignore === true) {
-        continue;
-      }
-      /**
-       * TODO: figure out correct types
-       */
-      // @ts-expect-error
-      const keys = unmarshall(dynamodb.Keys);
-      if (!keys?.PK || !keys.SK) {
-        continue;
-      }
-      const _id = `${keys.PK}:${keys.SK}`;
-      /**
-       * TODO: figure out correct types
-       */
-      // @ts-expect-error
-      const oldImage = unmarshall(dynamodb.OldImage);
-      const operation = record.eventName;
-
-      /**
-       * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.
-       * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.
-       */
-      let data = undefined;
-      if (newImage && operation !== Operations.REMOVE) {
-        /**
-         * We must decompress the data that is going into the Elasticsearch.
-         */
-        data = await (0, _apiElasticsearch.decompress)(context.plugins, newImage.data);
-        /**
-         * No point in writing null or undefined data into the Elasticsearch.
-         * This might happen on some error while decompressing. We will log it.
-         *
-         * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.
-         * If it is - it is a bug.
-         */
-        if (data === undefined || data === null) {
-          console.error(`Could not get decompressed data, skipping ES operation "${operation}", ID ${_id}`);
-          continue;
-        }
-      }
-      operationIdList.push(_id);
-      switch (record.eventName) {
-        case Operations.INSERT:
-        case Operations.MODIFY:
-          if (newImage) {
-            operations.push({
-              index: {
-                _id,
-                _index: newImage.index
-              }
-            }, data);
-          }
-          break;
-        case Operations.REMOVE:
-          operations.push({
-            delete: {
-              _id,
-              _index: oldImage?.index || "unknown"
-            }
-          });
-          break;
-        default:
-          break;
-      }
-    }
-    /**
-     * No need to do anything if there are no operations.
-     */
-    if (operations.length === 0) {
-      return null;
-    }
-    /**
-     * Wrap the code we need to run into the function, so it can be called within itself.
-     */
-    const execute = async () => {
-      const remainingTime = timer.getRemainingSeconds();
-      const runningTime = MAX_RUNNING_TIME - remainingTime;
-      const maxWaitingTime = remainingTime - 90;
-      if (process.env.DEBUG === "true") {
-        console.debug(`The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`);
-      }
-      const healthCheck = (0, _apiElasticsearch.createWaitUntilHealthy)(context.elasticsearch, {
-        minClusterHealthStatus: _types.ElasticsearchCatClusterHealthStatus.Yellow,
-        waitingTimeStep: 30,
-        maxProcessorPercent: MAX_PROCESSOR_PERCENT,
-        maxWaitingTime
-      });
-      try {
-        await healthCheck.wait({
-          async onUnhealthy({
-            startedAt,
-            runs,
-            mustEndAt,
-            waitingTimeStep,
-            waitingReason
-          }) {
-            console.debug(`Cluster is unhealthy on run #${runs}.`, {
-              startedAt,
-              mustEndAt,
-              waitingTimeStep,
-              waitingReason
-            });
-          },
-          async onTimeout({
-            startedAt,
-            runs,
-            waitingTimeStep,
-            mustEndAt,
-            waitingReason
-          }) {
-            console.error(`Cluster health check timeout on run #${runs}.`, {
-              startedAt,
-              mustEndAt,
-              waitingTimeStep,
-              waitingReason
-            });
-          }
-        });
-      } catch (ex) {
-        if (ex instanceof _apiElasticsearch.UnhealthyClusterError || ex instanceof _apiElasticsearch.WaitingHealthyClusterAbortedError) {
-          throw ex;
-        }
-        console.error(`Cluster health check failed.`, ex);
-        throw ex;
-      }
-      try {
-        const res = await context.elasticsearch.bulk({
-          body: operations
-        });
-        checkErrors(res);
-      } catch (error) {
-        if (process.env.DEBUG !== "true") {
-          throw error;
-        }
-        const meta = error?.meta || {};
-        delete meta["meta"];
-        console.error("Bulk error", JSON.stringify(error, null, 2));
-        throw error;
-      }
-      if (process.env.DEBUG !== "true") {
-        return;
-      }
-      console.info(`Transferred ${operations.length / 2} record operations to Elasticsearch.`);
-    };
-    const maxRetryTime = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME", 300000);
-    const retries = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES", 20);
-    const minTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT", 1500);
-    const maxTimeout = getNumberEnvVariable("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT", 30000);
-    try {
-      await (0, _pRetry.default)(execute, {
-        maxRetryTime,
-        retries,
-        minTimeout,
-        maxTimeout,
-        onFailedAttempt: error => {
-          if (timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {
-            throw new _NotEnoughRemainingTimeError.NotEnoughRemainingTimeError(error);
-          }
-          /**
-           * We will only log attempts which are after 3/4 of total attempts.
-           */
-          if (error.attemptNumber < retries * 0.75) {
-            return;
-          }
-          console.error(`Attempt #${error.attemptNumber} failed.`);
-          console.error(error);
-        }
-      });
-    } catch (ex) {
-      // TODO implement storing of failed operations
-      throw ex;
-    }
-    return null;
-  });
-};
-exports.createEventHandler = createEventHandler;
+export * from "./eventHandler.js";
+export * from "./execute.js";
+export * from "./executeWithRetry.js";
+export * from "./marshall.js";
+export * from "./NotEnoughRemainingTimeError.js";
+export * from "./Operations.js";
+export * from "./OperationsBuilder.js";
+export * from "./SynchronizationBuilder.js";
 
 //# sourceMappingURL=index.js.map

package/index.js.map

@@ -1 +1 @@
-{"version":3,"names":["_error","_interopRequireDefault","require","_clientDynamodb","_apiElasticsearch","_handlerAws","_types","_pRetry","_NotEnoughRemainingTimeError","Operations","exports","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","getNumberEnvVariable","name","def","input","value","Number","isNaN","MAX_PROCESSOR_PERCENT","NODE_ENV","unmarshall","undefined","baseUnmarshall","minRemainingSecondsToTimeout","MAX_RUNNING_TIME","createEventHandler","createDynamoDBEventHandler","event","context","ctx","lambdaContext","timer","timerFactory","elasticsearch","operations","operationIdList","record","Records","dynamodb","newImage","NewImage","ignore","keys","Keys","PK","SK","_id","oldImage","OldImage","operation","eventName","data","REMOVE","decompress","plugins","push","INSERT","MODIFY","_index","delete","length","execute","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","maxProcessorPercent","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","meta","info","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","onFailedAttempt","NotEnoughRemainingTimeError","attemptNumber"],"sources":["index.ts"],"sourcesContent":["import WebinyError from \"@webiny/error\";\nimport { AttributeValue, unmarshall as baseUnmarshall } from \"@webiny/aws-sdk/client-dynamodb\";\nimport {\n    createWaitUntilHealthy,\n    decompress,\n    UnhealthyClusterError,\n    WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport { ApiResponse, ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport { createDynamoDBEventHandler, timerFactory } from \"@webiny/handler-aws\";\nimport { ElasticsearchCatClusterHealthStatus } from \"@webiny/api-elasticsearch/operations/types\";\nimport pRetry from \"p-retry\";\nimport { NotEnoughRemainingTimeError } from \"./NotEnoughRemainingTimeError\";\n\nexport enum Operations {\n    INSERT = \"INSERT\",\n    MODIFY = \"MODIFY\",\n    REMOVE = \"REMOVE\"\n}\n\ninterface BulkOperationsResponseBodyItemIndexError {\n    reason?: string;\n}\n\ninterface BulkOperationsResponseBodyItemIndex {\n    error?: BulkOperationsResponseBodyItemIndexError;\n}\n\ninterface BulkOperationsResponseBodyItem {\n    index?: BulkOperationsResponseBodyItemIndex;\n    error?: string;\n}\n\ninterface BulkOperationsResponseBody {\n    items: BulkOperationsResponseBodyItem[];\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n    if (!item.index?.error?.reason) {\n        return null;\n    }\n    const reason = item.index.error.reason;\n    if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n        return \"index\";\n    }\n    return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n    if (!result || !result.body || !result.body.items) {\n        return;\n    }\n    for (const item of result.body.items) {\n        const err = getError(item);\n        if (!err) {\n            continue;\n        } else if (err === \"index\") {\n            if (process.env.DEBUG === \"true\") {\n                console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n            }\n            continue;\n        }\n        console.error(item.error);\n        throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n    }\n};\n\nconst getNumberEnvVariable = (name: string, def: number): number => {\n    const input = process.env[name];\n    const value = Number(input);\n    if (isNaN(value)) {\n        return def;\n    } else if (value <= 0) {\n        return def;\n    }\n    return value;\n};\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n    \"MAX_ES_PROCESSOR\",\n    process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\ninterface RecordDynamoDbImage {\n    data: Record<string, any>;\n    ignore?: boolean;\n    index: string;\n}\n\ninterface RecordDynamoDbKeys {\n    PK: string;\n    SK: string;\n}\n\nconst unmarshall = <T>(value?: Record<string, AttributeValue>): T | undefined => {\n    if (!value) {\n        return undefined;\n    }\n    return baseUnmarshall(value) as T;\n};\n\nconst minRemainingSecondsToTimeout = 120;\n\n/**\n * Also, we need to set the maximum running time for the Lambda Function.\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218\n */\nconst MAX_RUNNING_TIME = 900;\n\nexport const createEventHandler = () => {\n    return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => {\n        const timer = timerFactory(lambdaContext);\n        const context = ctx as unknown as ElasticsearchContext;\n        if (!context.elasticsearch) {\n            console.error(\"Missing elasticsearch definition on context.\");\n            return null;\n        }\n\n        const operations: Record<string, any>[] = [];\n\n        const operationIdList: string[] = [];\n\n        for (const record of event.Records) {\n            const dynamodb = record.dynamodb;\n            if (!dynamodb) {\n                continue;\n            }\n            /**\n             * TODO: figure out correct types\n             */\n            // @ts-expect-error\n            const newImage = unmarshall<RecordDynamoDbImage>(dynamodb.NewImage);\n\n            // Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n            // if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n            // need to process it.\n            if (newImage && newImage.ignore === true) {\n                continue;\n            }\n            /**\n             * TODO: figure out correct types\n             */\n            // @ts-expect-error\n            const keys = unmarshall<RecordDynamoDbKeys>(dynamodb.Keys);\n            if (!keys?.PK || !keys.SK) {\n                continue;\n            }\n            const _id = `${keys.PK}:${keys.SK}`;\n            /**\n             * TODO: figure out correct types\n             */\n            // @ts-expect-error\n            const oldImage = unmarshall<RecordDynamoDbImage>(dynamodb.OldImage);\n            const operation = record.eventName;\n\n            /**\n             * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n             * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n             */\n            let data: any = undefined;\n            if (newImage && operation !== Operations.REMOVE) {\n                /**\n                 * We must decompress the data that is going into the Elasticsearch.\n                 */\n                data = await decompress(context.plugins, newImage.data);\n                /**\n                 * No point in writing null or undefined data into the Elasticsearch.\n                 * This might happen on some error while decompressing. We will log it.\n                 *\n                 * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n                 * If it is - it is a bug.\n                 */\n                if (data === undefined || data === null) {\n                    console.error(\n                        `Could not get decompressed data, skipping ES operation \"${operation}\", ID ${_id}`\n                    );\n                    continue;\n                }\n            }\n\n            operationIdList.push(_id);\n\n            switch (record.eventName) {\n                case Operations.INSERT:\n                case Operations.MODIFY:\n                    if (newImage) {\n                        operations.push(\n                            {\n                                index: {\n                                    _id,\n                                    _index: newImage.index\n                                }\n                            },\n                            data\n                        );\n                    }\n                    break;\n                case Operations.REMOVE:\n                    operations.push({\n                        delete: {\n                            _id,\n                            _index: oldImage?.index || \"unknown\"\n                        }\n                    });\n                    break;\n                default:\n                    break;\n            }\n        }\n        /**\n         * No need to do anything if there are no operations.\n         */\n        if (operations.length === 0) {\n            return null;\n        }\n        /**\n         * Wrap the code we need to run into the function, so it can be called within itself.\n         */\n        const execute = async (): Promise<void> => {\n            const remainingTime = timer.getRemainingSeconds();\n            const runningTime = MAX_RUNNING_TIME - remainingTime;\n            const maxWaitingTime = remainingTime - 90;\n\n            if (process.env.DEBUG === \"true\") {\n                console.debug(\n                    `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n                );\n            }\n\n            const healthCheck = createWaitUntilHealthy(context.elasticsearch, {\n                minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,\n                waitingTimeStep: 30,\n                maxProcessorPercent: MAX_PROCESSOR_PERCENT,\n                maxWaitingTime\n            });\n\n            try {\n                await healthCheck.wait({\n                    async onUnhealthy({\n                        startedAt,\n                        runs,\n                        mustEndAt,\n                        waitingTimeStep,\n                        waitingReason\n                    }) {\n                        console.debug(`Cluster is unhealthy on run #${runs}.`, {\n                            startedAt,\n                            mustEndAt,\n                            waitingTimeStep,\n                            waitingReason\n                        });\n                    },\n                    async onTimeout({\n                        startedAt,\n                        runs,\n                        waitingTimeStep,\n                        mustEndAt,\n                        waitingReason\n                    }) {\n                        console.error(`Cluster health check timeout on run #${runs}.`, {\n                            startedAt,\n                            mustEndAt,\n                            waitingTimeStep,\n                            waitingReason\n                        });\n                    }\n                });\n            } catch (ex) {\n                if (\n                    ex instanceof UnhealthyClusterError ||\n                    ex instanceof WaitingHealthyClusterAbortedError\n                ) {\n                    throw ex;\n                }\n                console.error(`Cluster health check failed.`, ex);\n                throw ex;\n            }\n\n            try {\n                const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n                    body: operations\n                });\n                checkErrors(res);\n            } catch (error) {\n                if (process.env.DEBUG !== \"true\") {\n                    throw error;\n                }\n                const meta = error?.meta || {};\n                delete meta[\"meta\"];\n                console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n                throw error;\n            }\n            if (process.env.DEBUG !== \"true\") {\n                return;\n            }\n            console.info(\n                `Transferred ${operations.length / 2} record operations to Elasticsearch.`\n            );\n        };\n\n        const maxRetryTime = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n            300000\n        );\n        const retries = getNumberEnvVariable(\"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\", 20);\n        const minTimeout = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n            1500\n        );\n        const maxTimeout = getNumberEnvVariable(\n            \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n            30000\n        );\n\n        try {\n            await pRetry(execute, {\n                maxRetryTime,\n                retries,\n                minTimeout,\n                maxTimeout,\n                onFailedAttempt: error => {\n                    if (timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {\n                        throw new NotEnoughRemainingTimeError(error);\n                    }\n                    /**\n                     * We will only log attempts which are after 3/4 of total attempts.\n                     */\n                    if (error.attemptNumber < retries * 0.75) {\n                        return;\n                    }\n                    console.error(`Attempt #${error.attemptNumber} failed.`);\n                    console.error(error);\n                }\n            });\n        } catch (ex) {\n            // TODO implement storing of failed operations\n            throw ex;\n        }\n\n        return null;\n    });\n};\n"],"mappings":";;;;;;;AAAA,IAAAA,MAAA,GAAAC,sBAAA,CAAAC,OAAA;AACA,IAAAC,eAAA,GAAAD,OAAA;AACA,IAAAE,iBAAA,GAAAF,OAAA;AAOA,IAAAG,WAAA,GAAAH,OAAA;AACA,IAAAI,MAAA,GAAAJ,OAAA;AACA,IAAAK,OAAA,GAAAN,sBAAA,CAAAC,OAAA;AACA,IAAAM,4BAAA,GAAAN,OAAA;AAA4E,IAEhEO,UAAU,GAAAC,OAAA,CAAAD,UAAA,0BAAVA,UAAU;EAAVA,UAAU;EAAVA,UAAU;EAAVA,UAAU;EAAA,OAAVA,UAAU;AAAA;AAuBtB,MAAME,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,EAAEC,KAAK,EAAEC,MAAM,EAAE;IAC5B,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMR,IAAI,IAAIM,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACS,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACX,KAAK,CAAC,eAAe,EAAEY,IAAI,CAACC,SAAS,CAACT,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAO,OAAO,CAACX,KAAK,CAACF,IAAI,CAACE,KAAK,CAAC;IACzB,MAAM,IAAIc,cAAW,CAACP,GAAG,EAAE,iCAAiC,EAAET,IAAI,CAAC;EACvE;AACJ,CAAC;AAED,MAAMiB,oBAAoB,GAAGA,CAACC,IAAY,EAAEC,GAAW,KAAa;EAChE,MAAMC,KAAK,GAAGV,OAAO,CAACC,GAAG,CAACO,IAAI,CAAC;EAC/B,MAAMG,KAAK,GAAGC,MAAM,CAACF,KAAK,CAAC;EAC3B,IAAIG,KAAK,CAACF,KAAK,CAAC,EAAE;IACd,OAAOF,GAAG;EACd,CAAC,MAAM,IAAIE,KAAK,IAAI,CAAC,EAAE;IACnB,OAAOF,GAAG;EACd;EACA,OAAOE,KAAK;AAChB,CAAC;AAED,MAAMG,qBAAqB,GAAGP,oBAAoB,CAC9C,kBAAkB,EAClBP,OAAO,CAACC,GAAG,CAACc,QAAQ,KAAK,MAAM,GAAG,GAAG,GAAG,EAC5C,CAAC;AAaD,MAAMC,UAAU,GAAOL,KAAsC,IAAoB;EAC7E,IAAI,CAACA,KAAK,EAAE;IACR,OAAOM,SAAS;EACpB;EACA,OAAO,IAAAC,0BAAc,EAACP,KAAK,CAAC;AAChC,CAAC;AAED,MAAMQ,4BAA4B,GAAG,GAAG;;AAExC;AACA;AACA;AACA;AACA;AACA,MAAMC,gBAAgB,GAAG,GAAG;AAErB,MAAMC,kBAAkB,GAAGA,CAAA,KAAM;EACpC,OAAO,IAAAC,sCAA0B,EAAC,OAAO;IAAEC,KAAK;IAAEC,OAAO,EAAEC,GAAG;IAAEC;EAAc,CAAC,KAAK;IAChF,MAAMC,KAAK,GAAG,IAAAC,wBAAY,EAACF,aAAa,CAAC;IACzC,MAAMF,OAAO,GAAGC,GAAsC;IACtD,IAAI,CAACD,OAAO,CAACK,aAAa,EAAE;MACxB1B,OAAO,CAACX,KAAK,CAAC,8CAA8C,CAAC;MAC7D,OAAO,IAAI;IACf;IAEA,MAAMsC,UAAiC,GAAG,EAAE;IAE5C,MAAMC,eAAyB,GAAG,EAAE;IAEpC,KAAK,MAAMC,MAAM,IAAIT,KAAK,CAACU,OAAO,EAAE;MAChC,MAAMC,QAAQ,GAAGF,MAAM,CAACE,QAAQ;MAChC,IAAI,CAACA,QAAQ,EAAE;QACX;MACJ;MACA;AACZ;AACA;MACY;MACA,MAAMC,QAAQ,GAAGnB,UAAU,CAAsBkB,QAAQ,CAACE,QAAQ,CAAC;;MAEnE;MACA;MACA;MACA,IAAID,QAAQ,IAAIA,QAAQ,CAACE,MAAM,KAAK,IAAI,EAAE;QACtC;MACJ;MACA;AACZ;AACA;MACY;MACA,MAAMC,IAAI,GAAGtB,UAAU,CAAqBkB,QAAQ,CAACK,IAAI,CAAC;MAC1D,IAAI,CAACD,IAAI,EAAEE,EAAE,IAAI,CAACF,IAAI,CAACG,EAAE,EAAE;QACvB;MACJ;MACA,MAAMC,GAAG,GAAI,GAAEJ,IAAI,CAACE,EAAG,IAAGF,IAAI,CAACG,EAAG,EAAC;MACnC;AACZ;AACA;MACY;MACA,MAAME,QAAQ,GAAG3B,UAAU,CAAsBkB,QAAQ,CAACU,QAAQ,CAAC;MACnE,MAAMC,SAAS,GAAGb,MAAM,CAACc,SAAS;;MAElC;AACZ;AACA;AACA;MACY,IAAIC,IAAS,GAAG9B,SAAS;MACzB,IAAIkB,QAAQ,IAAIU,SAAS,KAAK1D,UAAU,CAAC6D,MAAM,EAAE;QAC7C;AAChB;AACA;QACgBD,IAAI,GAAG,MAAM,IAAAE,4BAAU,EAACzB,OAAO,CAAC0B,OAAO,EAAEf,QAAQ,CAACY,IAAI,CAAC;QACvD;AAChB;AACA;AACA;AACA;AACA;AACA;QACgB,IAAIA,IAAI,KAAK9B,SAAS,IAAI8B,IAAI,KAAK,IAAI,EAAE;UACrC5C,OAAO,CAACX,KAAK,CACR,2DAA0DqD,SAAU,SAAQH,GAAI,EACrF,CAAC;UACD;QACJ;MACJ;MAEAX,eAAe,CAACoB,IAAI,CAACT,GAAG,CAAC;MAEzB,QAAQV,MAAM,CAACc,SAAS;QACpB,KAAK3D,UAAU,CAACiE,MAAM;QACtB,KAAKjE,UAAU,CAACkE,MAAM;UAClB,IAAIlB,QAAQ,EAAE;YACVL,UAAU,CAACqB,IAAI,CACX;cACI5D,KAAK,EAAE;gBACHmD,GAAG;gBACHY,MAAM,EAAEnB,QAAQ,CAAC5C;cACrB;YACJ,CAAC,EACDwD,IACJ,CAAC;UACL;UACA;QACJ,KAAK5D,UAAU,CAAC6D,MAAM;UAClBlB,UAAU,CAACqB,IAAI,CAAC;YACZI,MAAM,EAAE;cACJb,GAAG;cACHY,MAAM,EAAEX,QAAQ,EAAEpD,KAAK,IAAI;YAC/B;UACJ,CAAC,CAAC;UACF;QACJ;UACI;MACR;IACJ;IACA;AACR;AACA;IACQ,IAAIuC,UAAU,CAAC0B,MAAM,KAAK,CAAC,EAAE;MACzB,OAAO,IAAI;IACf;IACA;AACR;AACA;IACQ,MAAMC,OAAO,GAAG,MAAAA,CAAA,KAA2B;MACvC,MAAMC,aAAa,GAAG/B,KAAK,CAACgC,mBAAmB,CAAC,CAAC;MACjD,MAAMC,WAAW,GAAGxC,gBAAgB,GAAGsC,aAAa;MACpD,MAAMG,cAAc,GAAGH,aAAa,GAAG,EAAE;MAEzC,IAAI1D,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAAC2D,KAAK,CACR,qCAAoCF,WAAY,6CAA4CC,cAAe,GAChH,CAAC;MACL;MAEA,MAAME,WAAW,GAAG,IAAAC,wCAAsB,EAACxC,OAAO,CAACK,aAAa,EAAE;QAC9DoC,sBAAsB,EAAEC,0CAAmC,CAACC,MAAM;QAClEC,eAAe,EAAE,EAAE;QACnBC,mBAAmB,EAAEvD,qBAAqB;QAC1C+C;MACJ,CAAC,CAAC;MAEF,IAAI;QACA,MAAME,WAAW,CAACO,IAAI,CAAC;UACnB,MAAMC,WAAWA,CAAC;YACdC,SAAS;YACTC,IAAI;YACJC,SAAS;YACTN,eAAe;YACfO;UACJ,CAAC,EAAE;YACCxE,OAAO,CAAC2D,KAAK,CAAE,gCAA+BW,IAAK,GAAE,EAAE;cACnDD,SAAS;cACTE,SAAS;cACTN,eAAe;cACfO;YACJ,CAAC,CAAC;UACN,CAAC;UACD,MAAMC,SAASA,CAAC;YACZJ,SAAS;YACTC,IAAI;YACJL,eAAe;YACfM,SAAS;YACTC;UACJ,CAAC,EAAE;YACCxE,OAAO,CAACX,KAAK,CAAE,wCAAuCiF,IAAK,GAAE,EAAE;cAC3DD,SAAS;cACTE,SAAS;cACTN,eAAe;cACfO;YACJ,CAAC,CAAC;UACN;QACJ,CAAC,CAAC;MACN,CAAC,CAAC,OAAOE,EAAE,EAAE;QACT,IACIA,EAAE,YAAYC,uCAAqB,IACnCD,EAAE,YAAYE,mDAAiC,EACjD;UACE,MAAMF,EAAE;QACZ;QACA1E,OAAO,CAACX,KAAK,CAAE,8BAA6B,EAAEqF,EAAE,CAAC;QACjD,MAAMA,EAAE;MACZ;MAEA,IAAI;QACA,MAAMG,GAAG,GAAG,MAAMxD,OAAO,CAACK,aAAa,CAACoD,IAAI,CAA6B;UACrEpF,IAAI,EAAEiC;QACV,CAAC,CAAC;QACFnC,WAAW,CAACqF,GAAG,CAAC;MACpB,CAAC,CAAC,OAAOxF,KAAK,EAAE;QACZ,IAAIQ,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9B,MAAMV,KAAK;QACf;QACA,MAAM0F,IAAI,GAAG1F,KAAK,EAAE0F,IAAI,IAAI,CAAC,CAAC;QAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;QACnB/E,OAAO,CAACX,KAAK,CAAC,YAAY,EAAEY,IAAI,CAACC,SAAS,CAACb,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3D,MAAMA,KAAK;MACf;MACA,IAAIQ,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9B;MACJ;MACAC,OAAO,CAACgF,IAAI,CACP,eAAcrD,UAAU,CAAC0B,MAAM,GAAG,CAAE,sCACzC,CAAC;IACL,CAAC;IAED,MAAM4B,YAAY,GAAG7E,oBAAoB,CACrC,iDAAiD,EACjD,MACJ,CAAC;IACD,MAAM8E,OAAO,GAAG9E,oBAAoB,CAAC,0CAA0C,EAAE,EAAE,CAAC;IACpF,MAAM+E,UAAU,GAAG/E,oBAAoB,CACnC,8CAA8C,EAC9C,IACJ,CAAC;IACD,MAAMgF,UAAU,GAAGhF,oBAAoB,CACnC,8CAA8C,EAC9C,KACJ,CAAC;IAED,IAAI;MACA,MAAM,IAAAiF,eAAM,EAAC/B,OAAO,EAAE;QAClB2B,YAAY;QACZC,OAAO;QACPC,UAAU;QACVC,UAAU;QACVE,eAAe,EAAEjG,KAAK,IAAI;UACtB,IAAImC,KAAK,CAACgC,mBAAmB,CAAC,CAAC,GAAGxC,4BAA4B,EAAE;YAC5D,MAAM,IAAIuE,wDAA2B,CAAClG,KAAK,CAAC;UAChD;UACA;AACpB;AACA;UACoB,IAAIA,KAAK,CAACmG,aAAa,GAAGN,OAAO,GAAG,IAAI,EAAE;YACtC;UACJ;UACAlF,OAAO,CAACX,KAAK,CAAE,YAAWA,KAAK,CAACmG,aAAc,UAAS,CAAC;UACxDxF,OAAO,CAACX,KAAK,CAACA,KAAK,CAAC;QACxB;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOqF,EAAE,EAAE;MACT;MACA,MAAMA,EAAE;IACZ;IAEA,OAAO,IAAI;EACf,CAAC,CAAC;AACN,CAAC;AAACzF,OAAA,CAAAiC,kBAAA,GAAAA,kBAAA","ignoreList":[]}
+{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./eventHandler.js\";\nexport * from \"./execute.js\";\nexport * from \"./executeWithRetry.js\";\nexport * from \"./marshall.js\";\nexport * from \"./NotEnoughRemainingTimeError.js\";\nexport * from \"./Operations.js\";\nexport * from \"./OperationsBuilder.js\";\nexport * from \"./SynchronizationBuilder.js\";\nexport type * from \"./types.js\";\n"],"mappings":"AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA","ignoreList":[]}

package/marshall.d.ts

@@ -0,0 +1,7 @@
+import type { GenericRecord } from "@webiny/api/types.js";
+import type { AttributeValue } from "@webiny/handler-aws/types.js";
+export interface MarshalledValue {
+    [key: string]: AttributeValue;
+}
+export declare const marshall: (value: GenericRecord) => MarshalledValue;
+export declare const unmarshall: <T>(value?: MarshalledValue) => T | undefined;

package/marshall.js

@@ -0,0 +1,19 @@
+import { marshall as baseMarshall, unmarshall as baseUnmarshall } from "@webiny/aws-sdk/client-dynamodb/index.js";
+export const marshall = value => {
+  if (!value) {
+    return value;
+  }
+  return baseMarshall(value);
+};
+export const unmarshall = value => {
+  if (!value) {
+    return undefined;
+  }
+  /**
+   * We can safely cast the return value to `T` because we are 100% positive that this is correct.
+   */
+  // @ts-expect-error
+  return baseUnmarshall(value);
+};
+
+//# sourceMappingURL=marshall.js.map

package/marshall.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["marshall","baseMarshall","unmarshall","baseUnmarshall","value","undefined"],"sources":["marshall.ts"],"sourcesContent":["import {\n    marshall as baseMarshall,\n    unmarshall as baseUnmarshall\n} from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport type { GenericRecord } from \"@webiny/api/types.js\";\n\nimport type { AttributeValue } from \"@webiny/handler-aws/types.js\";\n\nexport interface MarshalledValue {\n    [key: string]: AttributeValue;\n}\n\nexport const marshall = (value: GenericRecord): MarshalledValue => {\n    if (!value) {\n        return value;\n    }\n    return baseMarshall(value) as MarshalledValue;\n};\n\nexport const unmarshall = <T>(value?: MarshalledValue): T | undefined => {\n    if (!value) {\n        return undefined;\n    }\n    /**\n     * We can safely cast the return value to `T` because we are 100% positive that this is correct.\n     */\n    // @ts-expect-error\n    return baseUnmarshall(value) as T;\n};\n"],"mappings":"AAAA,SACIA,QAAQ,IAAIC,YAAY,EACxBC,UAAU,IAAIC,cAAc,QACzB,0CAA0C;AASjD,OAAO,MAAMH,QAAQ,GAAII,KAAoB,IAAsB;EAC/D,IAAI,CAACA,KAAK,EAAE;IACR,OAAOA,KAAK;EAChB;EACA,OAAOH,YAAY,CAACG,KAAK,CAAC;AAC9B,CAAC;AAED,OAAO,MAAMF,UAAU,GAAOE,KAAuB,IAAoB;EACrE,IAAI,CAACA,KAAK,EAAE;IACR,OAAOC,SAAS;EACpB;EACA;AACJ;AACA;EACI;EACA,OAAOF,cAAc,CAACC,KAAK,CAAC;AAChC,CAAC","ignoreList":[]}

package/package.json

@@ -1,6 +1,7 @@
 {
 	"name": "@webiny/api-dynamodb-to-elasticsearch",
-	"version": "6.0.0-beta.0",
+	"version": "6.0.0-rc.1",
+	"type": "module",
 	"main": "index.js",
 	"repository": {
 		"type": "git",
@@ -11,38 +12,27 @@
 	"license": "MIT",
 	"author": "Webiny Ltd.",
 	"dependencies": {
-		"@babel/runtime": "7.24.1",
-		"@webiny/api-elasticsearch": "6.0.0-beta.0",
-		"@webiny/aws-sdk": "6.0.0-beta.0",
-		"@webiny/error": "6.0.0-beta.0",
-		"@webiny/handler-aws": "6.0.0-beta.0",
-		"p-retry": "4.6.2"
+		"@webiny/api": "6.0.0-rc.1",
+		"@webiny/api-elasticsearch": "6.0.0-rc.1",
+		"@webiny/aws-sdk": "6.0.0-rc.1",
+		"@webiny/error": "6.0.0-rc.1",
+		"@webiny/handler-aws": "6.0.0-rc.1",
+		"@webiny/utils": "6.0.0-rc.1",
+		"p-retry": "7.1.1"
 	},
 	"devDependencies": {
-		"@babel/cli": "7.24.1",
-		"@babel/core": "7.24.3",
-		"@babel/plugin-proposal-object-rest-spread": "7.20.7",
-		"@babel/plugin-transform-runtime": "7.24.3",
-		"@babel/preset-env": "7.24.3",
-		"@babel/preset-typescript": "7.24.1",
-		"@types/aws-lambda": "8.10.136",
-		"@webiny/cli": "6.0.0-beta.0",
-		"@webiny/plugins": "6.0.0-beta.0",
-		"@webiny/project-utils": "6.0.0-beta.0",
-		"typescript": "4.7.4"
+		"@webiny/build-tools": "6.0.0-rc.1",
+		"@webiny/plugins": "6.0.0-rc.1",
+		"typescript": "5.9.3"
 	},
 	"publishConfig": {
 		"access": "public",
 		"directory": "dist"
 	},
-	"scripts": {
-		"build": "yarn webiny run build",
-		"watch": "yarn webiny run watch"
-	},
 	"adio": {
 		"ignoreDirs": [
 			"__tests__"
 		]
 	},
-	"gitHead": "aa8dbfbbd5ad13ec271adba6f2431e02991a300f"
+	"gitHead": "36d702721ff9ed39fb21d6f5fe7922a2a8716e63"
 }

package/types.d.ts

@@ -0,0 +1,29 @@
+import type { GenericRecord } from "@webiny/api/types.js";
+import type { DynamoDBRecord } from "@webiny/handler-aws/types.js";
+import type { ElasticsearchContext } from "@webiny/api-elasticsearch/types.js";
+export interface IOperationsBuilderBuildParams {
+    records: DynamoDBRecord[];
+}
+export interface IOperationsBuilder {
+    build(params: IOperationsBuilderBuildParams): Promise<IOperations>;
+}
+export interface IInsertOperationParams {
+    id: string;
+    index: string;
+    data: GenericRecord;
+}
+export type IModifyOperationParams = IInsertOperationParams;
+export interface IDeleteOperationParams {
+    id: string;
+    index: string;
+}
+export interface IOperations {
+    items: GenericRecord[];
+    total: number;
+    clear(): void;
+    insert(params: IInsertOperationParams): void;
+    modify(params: IModifyOperationParams): void;
+    delete(params: IDeleteOperationParams): void;
+}
+export interface Context extends ElasticsearchContext {
+}

package/types.js

@@ -0,0 +1,3 @@
+export {};
+
+//# sourceMappingURL=types.js.map

package/types.js.map

@@ -0,0 +1 @@
+{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { GenericRecord } from \"@webiny/api/types.js\";\nimport type { DynamoDBRecord } from \"@webiny/handler-aws/types.js\";\nimport type { ElasticsearchContext } from \"@webiny/api-elasticsearch/types.js\";\n\nexport interface IOperationsBuilderBuildParams {\n    records: DynamoDBRecord[];\n}\n\nexport interface IOperationsBuilder {\n    build(params: IOperationsBuilderBuildParams): Promise<IOperations>;\n}\n\nexport interface IInsertOperationParams {\n    id: string;\n    index: string;\n    data: GenericRecord;\n}\n\nexport type IModifyOperationParams = IInsertOperationParams;\n\nexport interface IDeleteOperationParams {\n    id: string;\n    index: string;\n}\n\nexport interface IOperations {\n    items: GenericRecord[];\n    total: number;\n    clear(): void;\n    insert(params: IInsertOperationParams): void;\n    modify(params: IModifyOperationParams): void;\n    delete(params: IDeleteOperationParams): void;\n}\n\nexport interface Context extends ElasticsearchContext {}\n"],"mappings":"","ignoreList":[]}