@webiny/api-dynamodb-to-elasticsearch 5.41.1 → 5.41.2-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Operations.d.ts +3 -1
- package/Operations.js +7 -1
- package/Operations.js.map +1 -1
- package/SynchronizationBuilder.d.ts +24 -0
- package/SynchronizationBuilder.js +46 -0
- package/SynchronizationBuilder.js.map +1 -0
- package/eventHandler.js +0 -4
- package/eventHandler.js.map +1 -1
- package/execute.d.ts +3 -4
- package/execute.js +3 -0
- package/execute.js.map +1 -1
- package/executeWithRetry.d.ts +2 -1
- package/executeWithRetry.js +2 -1
- package/executeWithRetry.js.map +1 -1
- package/index.d.ts +1 -0
- package/index.js +11 -0
- package/index.js.map +1 -1
- package/package.json +9 -9
- package/types.d.ts +3 -0
- package/types.js.map +1 -1
package/Operations.d.ts
CHANGED
|
@@ -6,8 +6,10 @@ export declare enum OperationType {
|
|
|
6
6
|
REMOVE = "REMOVE"
|
|
7
7
|
}
|
|
8
8
|
export declare class Operations implements IOperations {
|
|
9
|
-
|
|
9
|
+
private _items;
|
|
10
|
+
get items(): GenericRecord[];
|
|
10
11
|
get total(): number;
|
|
12
|
+
clear(): void;
|
|
11
13
|
insert(params: IInsertOperationParams): void;
|
|
12
14
|
modify(params: IModifyOperationParams): void;
|
|
13
15
|
delete(params: IDeleteOperationParams): void;
|
package/Operations.js
CHANGED
|
@@ -11,10 +11,16 @@ let OperationType = exports.OperationType = /*#__PURE__*/function (OperationType
|
|
|
11
11
|
return OperationType;
|
|
12
12
|
}({});
|
|
13
13
|
class Operations {
|
|
14
|
-
|
|
14
|
+
_items = [];
|
|
15
|
+
get items() {
|
|
16
|
+
return this._items;
|
|
17
|
+
}
|
|
15
18
|
get total() {
|
|
16
19
|
return this.items.length;
|
|
17
20
|
}
|
|
21
|
+
clear() {
|
|
22
|
+
this._items = [];
|
|
23
|
+
}
|
|
18
24
|
insert(params) {
|
|
19
25
|
this.items.push({
|
|
20
26
|
index: {
|
package/Operations.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["OperationType","exports","Operations","items","total","length","insert","params","push","index","_id","id","_index","data","modify","delete"],"sources":["Operations.ts"],"sourcesContent":["import { GenericRecord } from \"@webiny/cli/types\";\nimport {\n IDeleteOperationParams,\n IInsertOperationParams,\n IModifyOperationParams,\n IOperations\n} from \"~/types\";\n\nexport enum OperationType {\n INSERT = \"INSERT\",\n MODIFY = \"MODIFY\",\n REMOVE = \"REMOVE\"\n}\n\nexport class Operations implements IOperations {\n
|
|
1
|
+
{"version":3,"names":["OperationType","exports","Operations","_items","items","total","length","clear","insert","params","push","index","_id","id","_index","data","modify","delete"],"sources":["Operations.ts"],"sourcesContent":["import { GenericRecord } from \"@webiny/cli/types\";\nimport {\n IDeleteOperationParams,\n IInsertOperationParams,\n IModifyOperationParams,\n IOperations\n} from \"~/types\";\n\nexport enum OperationType {\n INSERT = \"INSERT\",\n MODIFY = \"MODIFY\",\n REMOVE = \"REMOVE\"\n}\n\nexport class Operations implements IOperations {\n private _items: GenericRecord[] = [];\n\n public get items(): GenericRecord[] {\n return this._items;\n }\n\n public get total(): number {\n return this.items.length;\n }\n\n public clear() {\n this._items = [];\n }\n\n public insert(params: IInsertOperationParams): void {\n this.items.push(\n {\n index: {\n _id: params.id,\n _index: params.index\n }\n },\n params.data\n );\n }\n\n public modify(params: IModifyOperationParams): void {\n this.insert(params);\n }\n\n public delete(params: IDeleteOperationParams): void {\n this.items.push({\n delete: {\n _id: params.id,\n _index: params.index\n }\n });\n }\n}\n"],"mappings":";;;;;;IAQYA,aAAa,GAAAC,OAAA,CAAAD,aAAA,0BAAbA,aAAa;EAAbA,aAAa;EAAbA,aAAa;EAAbA,aAAa;EAAA,OAAbA,aAAa;AAAA;AAMlB,MAAME,UAAU,CAAwB;EACnCC,MAAM,GAAoB,EAAE;EAEpC,IAAWC,KAAKA,CAAA,EAAoB;IAChC,OAAO,IAAI,CAACD,MAAM;EACtB;EAEA,IAAWE,KAAKA,CAAA,EAAW;IACvB,OAAO,IAAI,CAACD,KAAK,CAACE,MAAM;EAC5B;EAEOC,KAAKA,CAAA,EAAG;IACX,IAAI,CAACJ,MAAM,GAAG,EAAE;EACpB;EAEOK,MAAMA,CAACC,MAA8B,EAAQ;IAChD,IAAI,CAACL,KAAK,CAACM,IAAI,CACX;MACIC,KAAK,EAAE;QACHC,GAAG,EAAEH,MAAM,CAACI,EAAE;QACdC,MAAM,EAAEL,MAAM,CAACE;MACnB;IACJ,CAAC,EACDF,MAAM,CAACM,IACX,CAAC;EACL;EAEOC,MAAMA,CAACP,MAA8B,EAAQ;IAChD,IAAI,CAACD,MAAM,CAACC,MAAM,CAAC;EACvB;EAEOQ,MAAMA,CAACR,MAA8B,EAAQ;IAChD,IAAI,CAACL,KAAK,CAACM,IAAI,CAAC;MACZO,MAAM,EAAE;QACJL,GAAG,EAAEH,MAAM,CAACI,EAAE;QACdC,MAAM,EAAEL,MAAM,CAACE;MACnB;IACJ,CAAC,CAAC;EACN;AACJ;AAACV,OAAA,CAAAC,UAAA,GAAAA,UAAA","ignoreList":[]}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Context, IDeleteOperationParams, IInsertOperationParams, IModifyOperationParams } from "./types";
|
|
2
|
+
import { IExecuteWithRetryParams } from "./executeWithRetry";
|
|
3
|
+
import { ITimer } from "@webiny/handler-aws";
|
|
4
|
+
export type ISynchronizationBuilderExecuteWithRetryParams = Omit<IExecuteWithRetryParams, "context" | "timer" | "maxRunningTime" | "operations">;
|
|
5
|
+
export interface ISynchronizationBuilder {
|
|
6
|
+
insert(params: IInsertOperationParams): void;
|
|
7
|
+
delete(params: IDeleteOperationParams): void;
|
|
8
|
+
build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
export interface ISynchronizationBuilderParams {
|
|
11
|
+
timer: ITimer;
|
|
12
|
+
context: Pick<Context, "elasticsearch">;
|
|
13
|
+
}
|
|
14
|
+
export declare class SynchronizationBuilder implements ISynchronizationBuilder {
|
|
15
|
+
private readonly timer;
|
|
16
|
+
private readonly context;
|
|
17
|
+
private readonly operations;
|
|
18
|
+
constructor(params: ISynchronizationBuilderParams);
|
|
19
|
+
insert(params: IInsertOperationParams): void;
|
|
20
|
+
modify(params: IModifyOperationParams): void;
|
|
21
|
+
delete(params: IDeleteOperationParams): void;
|
|
22
|
+
build(): (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;
|
|
23
|
+
}
|
|
24
|
+
export declare const createSynchronizationBuilder: (params: ISynchronizationBuilderParams) => ISynchronizationBuilder;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.createSynchronizationBuilder = exports.SynchronizationBuilder = void 0;
|
|
7
|
+
var _Operations = require("./Operations");
|
|
8
|
+
var _executeWithRetry = require("./executeWithRetry");
|
|
9
|
+
class SynchronizationBuilder {
|
|
10
|
+
constructor(params) {
|
|
11
|
+
this.timer = params.timer;
|
|
12
|
+
this.context = params.context;
|
|
13
|
+
this.operations = new _Operations.Operations();
|
|
14
|
+
}
|
|
15
|
+
insert(params) {
|
|
16
|
+
return this.operations.insert(params);
|
|
17
|
+
}
|
|
18
|
+
modify(params) {
|
|
19
|
+
return this.operations.modify(params);
|
|
20
|
+
}
|
|
21
|
+
delete(params) {
|
|
22
|
+
return this.operations.delete(params);
|
|
23
|
+
}
|
|
24
|
+
build() {
|
|
25
|
+
return async params => {
|
|
26
|
+
if (this.operations.total === 0) {
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
await (0, _executeWithRetry.executeWithRetry)({
|
|
30
|
+
...params,
|
|
31
|
+
maxRunningTime: this.timer.getRemainingMilliseconds(),
|
|
32
|
+
timer: this.timer,
|
|
33
|
+
context: this.context,
|
|
34
|
+
operations: this.operations
|
|
35
|
+
});
|
|
36
|
+
this.operations.clear();
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
exports.SynchronizationBuilder = SynchronizationBuilder;
|
|
41
|
+
const createSynchronizationBuilder = params => {
|
|
42
|
+
return new SynchronizationBuilder(params);
|
|
43
|
+
};
|
|
44
|
+
exports.createSynchronizationBuilder = createSynchronizationBuilder;
|
|
45
|
+
|
|
46
|
+
//# sourceMappingURL=SynchronizationBuilder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["_Operations","require","_executeWithRetry","SynchronizationBuilder","constructor","params","timer","context","operations","Operations","insert","modify","delete","build","total","executeWithRetry","maxRunningTime","getRemainingMilliseconds","clear","exports","createSynchronizationBuilder"],"sources":["SynchronizationBuilder.ts"],"sourcesContent":["import {\n Context,\n IDeleteOperationParams,\n IInsertOperationParams,\n IModifyOperationParams,\n IOperations\n} from \"~/types\";\nimport { Operations } from \"~/Operations\";\nimport { executeWithRetry, IExecuteWithRetryParams } from \"~/executeWithRetry\";\nimport { ITimer } from \"@webiny/handler-aws\";\n\nexport type ISynchronizationBuilderExecuteWithRetryParams = Omit<\n IExecuteWithRetryParams,\n \"context\" | \"timer\" | \"maxRunningTime\" | \"operations\"\n>;\n\nexport interface ISynchronizationBuilder {\n insert(params: IInsertOperationParams): void;\n delete(params: IDeleteOperationParams): void;\n build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;\n}\n\nexport interface ISynchronizationBuilderParams {\n timer: ITimer;\n context: Pick<Context, \"elasticsearch\">;\n}\n\nexport class SynchronizationBuilder implements ISynchronizationBuilder {\n private readonly timer: ITimer;\n private readonly context: Pick<Context, \"elasticsearch\">;\n private readonly operations: IOperations;\n\n public constructor(params: ISynchronizationBuilderParams) {\n this.timer = params.timer;\n this.context = params.context;\n this.operations = new Operations();\n }\n\n public insert(params: IInsertOperationParams): void {\n return this.operations.insert(params);\n }\n\n public modify(params: IModifyOperationParams): void {\n return this.operations.modify(params);\n }\n\n public delete(params: IDeleteOperationParams): void {\n return this.operations.delete(params);\n }\n\n public build() {\n return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => {\n if (this.operations.total === 0) {\n return;\n }\n await executeWithRetry({\n ...params,\n maxRunningTime: this.timer.getRemainingMilliseconds(),\n timer: this.timer,\n context: this.context,\n operations: this.operations\n });\n this.operations.clear();\n };\n }\n}\n\nexport const createSynchronizationBuilder = (\n params: ISynchronizationBuilderParams\n): ISynchronizationBuilder => {\n return new SynchronizationBuilder(params);\n};\n"],"mappings":";;;;;;AAOA,IAAAA,WAAA,GAAAC,OAAA;AACA,IAAAC,iBAAA,GAAAD,OAAA;AAmBO,MAAME,sBAAsB,CAAoC;EAK5DC,WAAWA,CAACC,MAAqC,EAAE;IACtD,IAAI,CAACC,KAAK,GAAGD,MAAM,CAACC,KAAK;IACzB,IAAI,CAACC,OAAO,GAAGF,MAAM,CAACE,OAAO;IAC7B,IAAI,CAACC,UAAU,GAAG,IAAIC,sBAAU,CAAC,CAAC;EACtC;EAEOC,MAAMA,CAACL,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACE,MAAM,CAACL,MAAM,CAAC;EACzC;EAEOM,MAAMA,CAACN,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACG,MAAM,CAACN,MAAM,CAAC;EACzC;EAEOO,MAAMA,CAACP,MAA8B,EAAQ;IAChD,OAAO,IAAI,CAACG,UAAU,CAACI,MAAM,CAACP,MAAM,CAAC;EACzC;EAEOQ,KAAKA,CAAA,EAAG;IACX,OAAO,MAAOR,MAAsD,IAAK;MACrE,IAAI,IAAI,CAACG,UAAU,CAACM,KAAK,KAAK,CAAC,EAAE;QAC7B;MACJ;MACA,MAAM,IAAAC,kCAAgB,EAAC;QACnB,GAAGV,MAAM;QACTW,cAAc,EAAE,IAAI,CAACV,KAAK,CAACW,wBAAwB,CAAC,CAAC;QACrDX,KAAK,EAAE,IAAI,CAACA,KAAK;QACjBC,OAAO,EAAE,IAAI,CAACA,OAAO;QACrBC,UAAU,EAAE,IAAI,CAACA;MACrB,CAAC,CAAC;MACF,IAAI,CAACA,UAAU,CAACU,KAAK,CAAC,CAAC;IAC3B,CAAC;EACL;AACJ;AAACC,OAAA,CAAAhB,sBAAA,GAAAA,sBAAA;AAEM,MAAMiB,4BAA4B,GACrCf,MAAqC,IACX;EAC1B,OAAO,IAAIF,sBAAsB,CAACE,MAAM,CAAC;AAC7C,CAAC;AAACc,OAAA,CAAAC,4BAAA,GAAAA,4BAAA","ignoreList":[]}
|
package/eventHandler.js
CHANGED
|
@@ -4,13 +4,10 @@ Object.defineProperty(exports, "__esModule", {
|
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
6
|
exports.createEventHandler = void 0;
|
|
7
|
-
var _getNumberEnvVariable = require("./helpers/getNumberEnvVariable");
|
|
8
7
|
var _handlerAws = require("@webiny/handler-aws");
|
|
9
8
|
var _Decompressor = require("./Decompressor");
|
|
10
9
|
var _OperationsBuilder = require("./OperationsBuilder");
|
|
11
10
|
var _executeWithRetry = require("./executeWithRetry");
|
|
12
|
-
const MAX_PROCESSOR_PERCENT = (0, _getNumberEnvVariable.getNumberEnvVariable)("MAX_ES_PROCESSOR", process.env.NODE_ENV === "test" ? 101 : 98);
|
|
13
|
-
|
|
14
11
|
/**
|
|
15
12
|
* Also, we need to set the maximum running time for the Lambda Function.
|
|
16
13
|
* https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232
|
|
@@ -50,7 +47,6 @@ const createEventHandler = () => {
|
|
|
50
47
|
await (0, _executeWithRetry.executeWithRetry)({
|
|
51
48
|
timer,
|
|
52
49
|
maxRunningTime: MAX_RUNNING_TIME,
|
|
53
|
-
maxProcessorPercent: MAX_PROCESSOR_PERCENT,
|
|
54
50
|
context,
|
|
55
51
|
operations
|
|
56
52
|
});
|
package/eventHandler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["_handlerAws","require","_Decompressor","_OperationsBuilder","_executeWithRetry","MAX_RUNNING_TIME","createEventHandler","createDynamoDBEventHandler","event","context","ctx","lambdaContext","timer","timerFactory","elasticsearch","console","error","decompressor","Decompressor","plugins","builder","OperationsBuilder","operations","build","records","Records","total","executeWithRetry","maxRunningTime","exports"],"sources":["eventHandler.ts"],"sourcesContent":["import { createDynamoDBEventHandler, timerFactory } from \"@webiny/handler-aws\";\nimport { Context } from \"~/types\";\nimport { Decompressor } from \"~/Decompressor\";\nimport { OperationsBuilder } from \"~/OperationsBuilder\";\nimport { executeWithRetry } from \"~/executeWithRetry\";\n\n/**\n * Also, we need to set the maximum running time for the Lambda Function.\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218\n */\nconst MAX_RUNNING_TIME = 900;\n\nexport const createEventHandler = () => {\n return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => {\n const timer = timerFactory(lambdaContext);\n const context = ctx as unknown as Context;\n if (!context.elasticsearch) {\n console.error(\"Missing elasticsearch definition on context.\");\n return null;\n }\n\n const decompressor = new Decompressor({\n plugins: context.plugins\n });\n\n const builder = new OperationsBuilder({\n decompressor\n });\n\n const operations = await builder.build({\n records: event.Records\n });\n /**\n * No need to do anything if there are no operations.\n */\n if (operations.total === 0) {\n return null;\n }\n /**\n * Execute the operations with retry.\n */\n await executeWithRetry({\n timer,\n maxRunningTime: MAX_RUNNING_TIME,\n context,\n operations\n });\n\n return null;\n });\n};\n"],"mappings":";;;;;;AAAA,IAAAA,WAAA,GAAAC,OAAA;AAEA,IAAAC,aAAA,GAAAD,OAAA;AACA,IAAAE,kBAAA,GAAAF,OAAA;AACA,IAAAG,iBAAA,GAAAH,OAAA;AAEA;AACA;AACA;AACA;AACA;AACA,MAAMI,gBAAgB,GAAG,GAAG;AAErB,MAAMC,kBAAkB,GAAGA,CAAA,KAAM;EACpC,OAAO,IAAAC,sCAA0B,EAAC,OAAO;IAAEC,KAAK;IAAEC,OAAO,EAAEC,GAAG;IAAEC;EAAc,CAAC,KAAK;IAChF,MAAMC,KAAK,GAAG,IAAAC,wBAAY,EAACF,aAAa,CAAC;IACzC,MAAMF,OAAO,GAAGC,GAAyB;IACzC,IAAI,CAACD,OAAO,CAACK,aAAa,EAAE;MACxBC,OAAO,CAACC,KAAK,CAAC,8CAA8C,CAAC;MAC7D,OAAO,IAAI;IACf;IAEA,MAAMC,YAAY,GAAG,IAAIC,0BAAY,CAAC;MAClCC,OAAO,EAAEV,OAAO,CAACU;IACrB,CAAC,CAAC;IAEF,MAAMC,OAAO,GAAG,IAAIC,oCAAiB,CAAC;MAClCJ;IACJ,CAAC,CAAC;IAEF,MAAMK,UAAU,GAAG,MAAMF,OAAO,CAACG,KAAK,CAAC;MACnCC,OAAO,EAAEhB,KAAK,CAACiB;IACnB,CAAC,CAAC;IACF;AACR;AACA;IACQ,IAAIH,UAAU,CAACI,KAAK,KAAK,CAAC,EAAE;MACxB,OAAO,IAAI;IACf;IACA;AACR;AACA;IACQ,MAAM,IAAAC,kCAAgB,EAAC;MACnBf,KAAK;MACLgB,cAAc,EAAEvB,gBAAgB;MAChCI,OAAO;MACPa;IACJ,CAAC,CAAC;IAEF,OAAO,IAAI;EACf,CAAC,CAAC;AACN,CAAC;AAACO,OAAA,CAAAvB,kBAAA,GAAAA,kBAAA","ignoreList":[]}
|
package/execute.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { ITimer } from "@webiny/handler-aws";
|
|
2
|
-
import {
|
|
3
|
-
import { IOperations } from "./types";
|
|
2
|
+
import { Context, IOperations } from "./types";
|
|
4
3
|
export interface BulkOperationsResponseBodyItemIndexError {
|
|
5
4
|
reason?: string;
|
|
6
5
|
}
|
|
@@ -18,7 +17,7 @@ export interface IExecuteParams {
|
|
|
18
17
|
timer: ITimer;
|
|
19
18
|
maxRunningTime: number;
|
|
20
19
|
maxProcessorPercent: number;
|
|
21
|
-
context: Pick<
|
|
22
|
-
operations: IOperations
|
|
20
|
+
context: Pick<Context, "elasticsearch">;
|
|
21
|
+
operations: Pick<IOperations, "items" | "total">;
|
|
23
22
|
}
|
|
24
23
|
export declare const execute: (params: IExecuteParams) => () => Promise<void>;
|
package/execute.js
CHANGED
|
@@ -43,6 +43,9 @@ const execute = params => {
|
|
|
43
43
|
maxProcessorPercent,
|
|
44
44
|
operations
|
|
45
45
|
} = params;
|
|
46
|
+
if (operations.total === 0) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
46
49
|
const remainingTime = timer.getRemainingSeconds();
|
|
47
50
|
const runningTime = maxRunningTime - remainingTime;
|
|
48
51
|
const maxWaitingTime = remainingTime - 90;
|
package/execute.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_apiElasticsearch","require","_error","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","elasticsearch","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","meta","info","
|
|
1
|
+
{"version":3,"names":["_apiElasticsearch","require","_error","getError","item","index","error","reason","match","checkErrors","result","body","items","err","process","env","DEBUG","console","JSON","stringify","WebinyError","execute","params","context","timer","maxRunningTime","maxProcessorPercent","operations","total","remainingTime","getRemainingSeconds","runningTime","maxWaitingTime","debug","healthCheck","createWaitUntilHealthy","elasticsearch","minClusterHealthStatus","ElasticsearchCatClusterHealthStatus","Yellow","waitingTimeStep","wait","onUnhealthy","startedAt","runs","mustEndAt","waitingReason","onTimeout","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","bulk","meta","info","exports"],"sources":["execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n ElasticsearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-elasticsearch\";\nimport { ITimer } from \"@webiny/handler-aws\";\nimport { ApiResponse } from \"@webiny/api-elasticsearch/types\";\nimport { WebinyError } from \"@webiny/error\";\nimport { Context, IOperations } from \"./types\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n timer: ITimer;\n maxRunningTime: number;\n maxProcessorPercent: number;\n context: Pick<Context, \"elasticsearch\">;\n operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index?.error?.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse<BulkOperationsResponseBody>): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(item.error);\n throw new WebinyError(err, \"DYNAMODB_TO_ELASTICSEARCH_ERROR\", item);\n }\n};\n\nexport const execute = (params: IExecuteParams) => {\n return async (): Promise<void> => {\n const { context, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n if (operations.total === 0) {\n return;\n }\n\n const remainingTime = timer.getRemainingSeconds();\n const runningTime = maxRunningTime - remainingTime;\n const maxWaitingTime = remainingTime - 90;\n\n if (process.env.DEBUG === \"true\") {\n console.debug(\n `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n );\n }\n\n const healthCheck = createWaitUntilHealthy(context.elasticsearch, {\n minClusterHealthStatus: ElasticsearchCatClusterHealthStatus.Yellow,\n waitingTimeStep: 30,\n maxProcessorPercent,\n maxWaitingTime\n });\n\n try {\n await healthCheck.wait({\n async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n console.debug(`Cluster is unhealthy on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n },\n async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n console.error(`Cluster health check timeout on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n }\n });\n } catch (ex) {\n if (\n ex instanceof UnhealthyClusterError ||\n ex instanceof WaitingHealthyClusterAbortedError\n ) {\n throw ex;\n }\n console.error(`Cluster health check failed.`, ex);\n throw ex;\n }\n\n try {\n const res = await context.elasticsearch.bulk<BulkOperationsResponseBody>({\n body: operations.items\n });\n checkErrors(res);\n } catch (error) {\n if (process.env.DEBUG !== \"true\") {\n throw error;\n }\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n throw error;\n }\n if (process.env.DEBUG !== \"true\") {\n return;\n }\n console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n };\n};\n"],"mappings":";;;;;;AAAA,IAAAA,iBAAA,GAAAC,OAAA;AAQA,IAAAC,MAAA,GAAAD,OAAA;AA4BA,MAAME,QAAQ,GAAIC,IAAoC,IAAoB;EACtE,IAAI,CAACA,IAAI,CAACC,KAAK,EAAEC,KAAK,EAAEC,MAAM,EAAE;IAC5B,OAAO,IAAI;EACf;EACA,MAAMA,MAAM,GAAGH,IAAI,CAACC,KAAK,CAACC,KAAK,CAACC,MAAM;EACtC,IAAIA,MAAM,CAACC,KAAK,CAAC,oCAAoC,CAAC,KAAK,IAAI,EAAE;IAC7D,OAAO,OAAO;EAClB;EACA,OAAOD,MAAM;AACjB,CAAC;AAED,MAAME,WAAW,GAAIC,MAAgD,IAAW;EAC5E,IAAI,CAACA,MAAM,IAAI,CAACA,MAAM,CAACC,IAAI,IAAI,CAACD,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAC/C;EACJ;EACA,KAAK,MAAMR,IAAI,IAAIM,MAAM,CAACC,IAAI,CAACC,KAAK,EAAE;IAClC,MAAMC,GAAG,GAAGV,QAAQ,CAACC,IAAI,CAAC;IAC1B,IAAI,CAACS,GAAG,EAAE;MACN;IACJ,CAAC,MAAM,IAAIA,GAAG,KAAK,OAAO,EAAE;MACxB,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9BC,OAAO,CAACX,KAAK,CAAC,eAAe,EAAEY,IAAI,CAACC,SAAS,CAACT,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MACnE;MACA;IACJ;IACAO,OAAO,CAACX,KAAK,CAACF,IAAI,CAACE,KAAK,CAAC;IACzB,MAAM,IAAIc,kBAAW,CAACP,GAAG,EAAE,iCAAiC,EAAET,IAAI,CAAC;EACvE;AACJ,CAAC;AAEM,MAAMiB,OAAO,GAAIC,MAAsB,IAAK;EAC/C,OAAO,YAA2B;IAC9B,MAAM;MAAEC,OAAO;MAAEC,KAAK;MAAEC,cAAc;MAAEC,mBAAmB;MAAEC;IAAW,CAAC,GAAGL,MAAM;IAElF,IAAIK,UAAU,CAACC,KAAK,KAAK,CAAC,EAAE;MACxB;IACJ;IAEA,MAAMC,aAAa,GAAGL,KAAK,CAACM,mBAAmB,CAAC,CAAC;IACjD,MAAMC,WAAW,GAAGN,cAAc,GAAGI,aAAa;IAClD,MAAMG,cAAc,GAAGH,aAAa,GAAG,EAAE;IAEzC,IAAIf,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9BC,OAAO,CAACgB,KAAK,CACR,qCAAoCF,WAAY,6CAA4CC,cAAe,GAChH,CAAC;IACL;IAEA,MAAME,WAAW,GAAG,IAAAC,wCAAsB,EAACZ,OAAO,CAACa,aAAa,EAAE;MAC9DC,sBAAsB,EAAEC,qDAAmC,CAACC,MAAM;MAClEC,eAAe,EAAE,EAAE;MACnBd,mBAAmB;MACnBM;IACJ,CAAC,CAAC;IAEF,IAAI;MACA,MAAME,WAAW,CAACO,IAAI,CAAC;QACnB,MAAMC,WAAWA,CAAC;UAAEC,SAAS;UAAEC,IAAI;UAAEC,SAAS;UAAEL,eAAe;UAAEM;QAAc,CAAC,EAAE;UAC9E7B,OAAO,CAACgB,KAAK,CAAE,gCAA+BW,IAAK,GAAE,EAAE;YACnDD,SAAS;YACTE,SAAS;YACTL,eAAe;YACfM;UACJ,CAAC,CAAC;QACN,CAAC;QACD,MAAMC,SAASA,CAAC;UAAEJ,SAAS;UAAEC,IAAI;UAAEJ,eAAe;UAAEK,SAAS;UAAEC;QAAc,CAAC,EAAE;UAC5E7B,OAAO,CAACX,KAAK,CAAE,wCAAuCsC,IAAK,GAAE,EAAE;YAC3DD,SAAS;YACTE,SAAS;YACTL,eAAe;YACfM;UACJ,CAAC,CAAC;QACN;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,IACIA,EAAE,YAAYC,uCAAqB,IACnCD,EAAE,YAAYE,mDAAiC,EACjD;QACE,MAAMF,EAAE;MACZ;MACA/B,OAAO,CAACX,KAAK,CAAE,8BAA6B,EAAE0C,EAAE,CAAC;MACjD,MAAMA,EAAE;IACZ;IAEA,IAAI;MACA,MAAMG,GAAG,GAAG,MAAM5B,OAAO,CAACa,aAAa,CAACgB,IAAI,CAA6B;QACrEzC,IAAI,EAAEgB,UAAU,CAACf;MACrB,CAAC,CAAC;MACFH,WAAW,CAAC0C,GAAG,CAAC;IACpB,CAAC,CAAC,OAAO7C,KAAK,EAAE;MACZ,IAAIQ,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;QAC9B,MAAMV,KAAK;MACf;MACA,MAAM+C,IAAI,GAAG/C,KAAK,EAAE+C,IAAI,IAAI,CAAC,CAAC;MAC9B,OAAOA,IAAI,CAAC,MAAM,CAAC;MACnBpC,OAAO,CAACX,KAAK,CAAC,YAAY,EAAEY,IAAI,CAACC,SAAS,CAACb,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;MAC3D,MAAMA,KAAK;IACf;IACA,IAAIQ,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;MAC9B;IACJ;IACAC,OAAO,CAACqC,IAAI,CAAE,eAAc3B,UAAU,CAACC,KAAM,sCAAqC,CAAC;EACvF,CAAC;AACL,CAAC;AAAC2B,OAAA,CAAAlC,OAAA,GAAAA,OAAA","ignoreList":[]}
|
package/executeWithRetry.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { IExecuteParams } from "./execute";
|
|
2
|
-
export interface IExecuteWithRetryParams extends IExecuteParams {
|
|
2
|
+
export interface IExecuteWithRetryParams extends Omit<IExecuteParams, "maxProcessorPercent"> {
|
|
3
3
|
maxRetryTime?: number;
|
|
4
4
|
retries?: number;
|
|
5
5
|
minTimeout?: number;
|
|
6
6
|
maxTimeout?: number;
|
|
7
|
+
maxProcessorPercent?: number;
|
|
7
8
|
}
|
|
8
9
|
export declare const executeWithRetry: (params: IExecuteWithRetryParams) => Promise<void>;
|
package/executeWithRetry.js
CHANGED
|
@@ -10,6 +10,7 @@ var _NotEnoughRemainingTimeError = require("./NotEnoughRemainingTimeError");
|
|
|
10
10
|
var _pRetry = _interopRequireDefault(require("p-retry"));
|
|
11
11
|
var _getNumberEnvVariable = require("./helpers/getNumberEnvVariable");
|
|
12
12
|
const minRemainingSecondsToTimeout = 120;
|
|
13
|
+
const MAX_PROCESSOR_PERCENT = (0, _getNumberEnvVariable.getNumberEnvVariable)("MAX_ES_PROCESSOR", process.env.NODE_ENV === "test" ? 101 : 98);
|
|
13
14
|
const executeWithRetry = async params => {
|
|
14
15
|
const maxRetryTime = (0, _getNumberEnvVariable.getNumberEnvVariable)("WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME", params.maxRetryTime || 300000);
|
|
15
16
|
const retries = (0, _getNumberEnvVariable.getNumberEnvVariable)("WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES", params.retries || 20);
|
|
@@ -19,7 +20,7 @@ const executeWithRetry = async params => {
|
|
|
19
20
|
await (0, _pRetry.default)((0, _execute.execute)({
|
|
20
21
|
timer: params.timer,
|
|
21
22
|
maxRunningTime: params.maxRunningTime,
|
|
22
|
-
maxProcessorPercent: params.maxProcessorPercent,
|
|
23
|
+
maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,
|
|
23
24
|
context: params.context,
|
|
24
25
|
operations: params.operations
|
|
25
26
|
}), {
|
package/executeWithRetry.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_execute","require","_NotEnoughRemainingTimeError","_pRetry","_interopRequireDefault","_getNumberEnvVariable","minRemainingSecondsToTimeout","executeWithRetry","params","maxRetryTime","
|
|
1
|
+
{"version":3,"names":["_execute","require","_NotEnoughRemainingTimeError","_pRetry","_interopRequireDefault","_getNumberEnvVariable","minRemainingSecondsToTimeout","MAX_PROCESSOR_PERCENT","getNumberEnvVariable","process","env","NODE_ENV","executeWithRetry","params","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","execute","timer","maxRunningTime","maxProcessorPercent","context","operations","onFailedAttempt","error","getRemainingSeconds","NotEnoughRemainingTimeError","attemptNumber","console","ex","exports"],"sources":["executeWithRetry.ts"],"sourcesContent":["import { execute, IExecuteParams } from \"~/execute\";\nimport { NotEnoughRemainingTimeError } from \"~/NotEnoughRemainingTimeError\";\nimport pRetry from \"p-retry\";\nimport { getNumberEnvVariable } from \"./helpers/getNumberEnvVariable\";\n\nconst minRemainingSecondsToTimeout = 120;\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n \"MAX_ES_PROCESSOR\",\n process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\nexport interface IExecuteWithRetryParams extends Omit<IExecuteParams, \"maxProcessorPercent\"> {\n maxRetryTime?: number;\n retries?: number;\n minTimeout?: number;\n maxTimeout?: number;\n maxProcessorPercent?: number;\n}\n\nexport const executeWithRetry = async (params: IExecuteWithRetryParams) => {\n const maxRetryTime = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_RETRY_TIME\",\n params.maxRetryTime || 300000\n );\n const retries = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_RETRIES\",\n params.retries || 20\n );\n const minTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MIN_TIMEOUT\",\n params.minTimeout || 1500\n );\n const maxTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_ELASTICSEARCH_MAX_TIMEOUT\",\n params.maxTimeout || 30000\n );\n\n try {\n await pRetry(\n execute({\n timer: params.timer,\n maxRunningTime: params.maxRunningTime,\n maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,\n context: params.context,\n operations: params.operations\n }),\n {\n maxRetryTime,\n retries,\n minTimeout,\n maxTimeout,\n onFailedAttempt: error => {\n if (params.timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {\n throw new NotEnoughRemainingTimeError(error);\n }\n /**\n * We will only log attempts which are after 3/4 of total attempts.\n */\n if (error.attemptNumber < retries * 0.75) {\n return;\n }\n console.error(`Attempt #${error.attemptNumber} failed.`);\n console.error(error);\n }\n }\n );\n } catch (ex) {\n // TODO implement storing of failed operations\n throw ex;\n }\n};\n"],"mappings":";;;;;;;AAAA,IAAAA,QAAA,GAAAC,OAAA;AACA,IAAAC,4BAAA,GAAAD,OAAA;AACA,IAAAE,OAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,qBAAA,GAAAJ,OAAA;AAEA,MAAMK,4BAA4B,GAAG,GAAG;AAExC,MAAMC,qBAAqB,GAAG,IAAAC,0CAAoB,EAC9C,kBAAkB,EAClBC,OAAO,CAACC,GAAG,CAACC,QAAQ,KAAK,MAAM,GAAG,GAAG,GAAG,EAC5C,CAAC;AAUM,MAAMC,gBAAgB,GAAG,MAAOC,MAA+B,IAAK;EACvE,MAAMC,YAAY,GAAG,IAAAN,0CAAoB,EACrC,iDAAiD,EACjDK,MAAM,CAACC,YAAY,IAAI,MAC3B,CAAC;EACD,MAAMC,OAAO,GAAG,IAAAP,0CAAoB,EAChC,0CAA0C,EAC1CK,MAAM,CAACE,OAAO,IAAI,EACtB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAR,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACG,UAAU,IAAI,IACzB,CAAC;EACD,MAAMC,UAAU,GAAG,IAAAT,0CAAoB,EACnC,8CAA8C,EAC9CK,MAAM,CAACI,UAAU,IAAI,KACzB,CAAC;EAED,IAAI;IACA,MAAM,IAAAC,eAAM,EACR,IAAAC,gBAAO,EAAC;MACJC,KAAK,EAAEP,MAAM,CAACO,KAAK;MACnBC,cAAc,EAAER,MAAM,CAACQ,cAAc;MACrCC,mBAAmB,EAAET,MAAM,CAACS,mBAAmB,IAAIf,qBAAqB;MACxEgB,OAAO,EAAEV,MAAM,CAACU,OAAO;MACvBC,UAAU,EAAEX,MAAM,CAACW;IACvB,CAAC,CAAC,EACF;MACIV,YAAY;MACZC,OAAO;MACPC,UAAU;MACVC,UAAU;MACVQ,eAAe,EAAEC,KAAK,IAAI;QACtB,IAAIb,MAAM,CAACO,KAAK,CAACO,mBAAmB,CAAC,CAAC,GAAGrB,4BAA4B,EAAE;UACnE,MAAM,IAAIsB,wDAA2B,CAACF,KAAK,CAAC;QAChD;QACA;AACpB;AACA;QACoB,IAAIA,KAAK,CAACG,aAAa,GAAGd,OAAO,GAAG,IAAI,EAAE;UACtC;QACJ;QACAe,OAAO,CAACJ,KAAK,CAAE,YAAWA,KAAK,CAACG,aAAc,UAAS,CAAC;QACxDC,OAAO,CAACJ,KAAK,CAACA,KAAK,CAAC;MACxB;IACJ,CACJ,CAAC;EACL,CAAC,CAAC,OAAOK,EAAE,EAAE;IACT;IACA,MAAMA,EAAE;EACZ;AACJ,CAAC;AAACC,OAAA,CAAApB,gBAAA,GAAAA,gBAAA","ignoreList":[]}
|
package/index.d.ts
CHANGED
package/index.js
CHANGED
|
@@ -91,6 +91,17 @@ Object.keys(_OperationsBuilder).forEach(function (key) {
|
|
|
91
91
|
}
|
|
92
92
|
});
|
|
93
93
|
});
|
|
94
|
+
var _SynchronizationBuilder = require("./SynchronizationBuilder");
|
|
95
|
+
Object.keys(_SynchronizationBuilder).forEach(function (key) {
|
|
96
|
+
if (key === "default" || key === "__esModule") return;
|
|
97
|
+
if (key in exports && exports[key] === _SynchronizationBuilder[key]) return;
|
|
98
|
+
Object.defineProperty(exports, key, {
|
|
99
|
+
enumerable: true,
|
|
100
|
+
get: function () {
|
|
101
|
+
return _SynchronizationBuilder[key];
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
});
|
|
94
105
|
var _types = require("./types");
|
|
95
106
|
Object.keys(_types).forEach(function (key) {
|
|
96
107
|
if (key === "default" || key === "__esModule") return;
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_Decompressor","require","Object","keys","forEach","key","exports","defineProperty","enumerable","get","_eventHandler","_execute","_executeWithRetry","_marshall","_NotEnoughRemainingTimeError","_Operations","_OperationsBuilder","_types"],"sources":["index.ts"],"sourcesContent":["export * from \"./Decompressor\";\nexport * from \"./eventHandler\";\nexport * from \"./execute\";\nexport * from \"./executeWithRetry\";\nexport * from \"./marshall\";\nexport * from \"./NotEnoughRemainingTimeError\";\nexport * from \"./Operations\";\nexport * from \"./OperationsBuilder\";\nexport * from \"./types\";\n"],"mappings":";;;;;AAAA,IAAAA,aAAA,GAAAC,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAH,aAAA,EAAAI,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAL,aAAA,CAAAK,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAT,aAAA,CAAAK,GAAA;IAAA;EAAA;AAAA;AACA,IAAAK,aAAA,GAAAT,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAO,aAAA,EAAAN,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAK,aAAA,CAAAL,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAC,aAAA,CAAAL,GAAA;IAAA;EAAA;AAAA;AACA,IAAAM,QAAA,GAAAV,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAQ,QAAA,EAAAP,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAM,QAAA,CAAAN,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAE,QAAA,CAAAN,GAAA;IAAA;EAAA;AAAA;AACA,IAAAO,iBAAA,GAAAX,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAS,iBAAA,EAAAR,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAO,iBAAA,CAAAP,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAG,iBAAA,CAAAP,GAAA;IAAA;EAAA;AAAA;AACA,IAAAQ,SAAA,GAAAZ,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAU,SAAA,EAAAT,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAQ,SAAA,CAAAR,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAI,SAAA,CAAAR,GAAA;IAAA;EAAA;AAAA;AACA,IAAAS,4BAAA,GAAAb,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAW,4BAAA,EAAAV,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAS,4BAAA,CAAAT,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAK,4BAAA,CAAAT,GAAA;IAAA;EAAA;AAAA;AACA,IAAAU,WAAA,GAAAd,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAY,WAAA,EAAAX,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAU,WAAA,CAAAV,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAM,WAAA,CAAAV,GAAA;IAAA;EAAA;AAAA;AACA,IAAAW,kBAAA,GAAAf,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAa,kBAAA,EAAAZ,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAW,kBAAA,CAAAX,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAO,kBAAA,CAAAX,GAAA;IAAA;EAAA;AAAA;AACA,IAAAY,
|
|
1
|
+
{"version":3,"names":["_Decompressor","require","Object","keys","forEach","key","exports","defineProperty","enumerable","get","_eventHandler","_execute","_executeWithRetry","_marshall","_NotEnoughRemainingTimeError","_Operations","_OperationsBuilder","_SynchronizationBuilder","_types"],"sources":["index.ts"],"sourcesContent":["export * from \"./Decompressor\";\nexport * from \"./eventHandler\";\nexport * from \"./execute\";\nexport * from \"./executeWithRetry\";\nexport * from \"./marshall\";\nexport * from \"./NotEnoughRemainingTimeError\";\nexport * from \"./Operations\";\nexport * from \"./OperationsBuilder\";\nexport * from \"./SynchronizationBuilder\";\nexport * from \"./types\";\n"],"mappings":";;;;;AAAA,IAAAA,aAAA,GAAAC,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAH,aAAA,EAAAI,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAL,aAAA,CAAAK,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAT,aAAA,CAAAK,GAAA;IAAA;EAAA;AAAA;AACA,IAAAK,aAAA,GAAAT,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAO,aAAA,EAAAN,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAK,aAAA,CAAAL,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAC,aAAA,CAAAL,GAAA;IAAA;EAAA;AAAA;AACA,IAAAM,QAAA,GAAAV,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAQ,QAAA,EAAAP,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAM,QAAA,CAAAN,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAE,QAAA,CAAAN,GAAA;IAAA;EAAA;AAAA;AACA,IAAAO,iBAAA,GAAAX,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAS,iBAAA,EAAAR,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAO,iBAAA,CAAAP,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAG,iBAAA,CAAAP,GAAA;IAAA;EAAA;AAAA;AACA,IAAAQ,SAAA,GAAAZ,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAU,SAAA,EAAAT,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAQ,SAAA,CAAAR,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAI,SAAA,CAAAR,GAAA;IAAA;EAAA;AAAA;AACA,IAAAS,4BAAA,GAAAb,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAW,4BAAA,EAAAV,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAS,4BAAA,CAAAT,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAK,4BAAA,CAAAT,GAAA;IAAA;EAAA;AAAA;AACA,IAAAU,WAAA,GAAAd,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAY,WAAA,EAAAX,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAU,WAAA,CAAAV,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAM,WAAA,CAAAV,GAAA;IAAA;EAAA;AAAA;AACA,IAAAW,kBAAA,GAAAf,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAa,kBAAA,EAAAZ,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAW,kBAAA,CAAAX,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAO,kBAAA,CAAAX,GAAA;IAAA;EAAA;AAAA;AACA,IAAAY,uBAAA,GAAAhB,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAc,uBAAA,EAAAb,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAY,uBAAA,CAAAZ,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAQ,uBAAA,CAAAZ,GAAA;IAAA;EAAA;AAAA;AACA,IAAAa,MAAA,GAAAjB,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAe,MAAA,EAAAd,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAa,MAAA,CAAAb,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAS,MAAA,CAAAb,GAAA;IAAA;EAAA;AAAA","ignoreList":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webiny/api-dynamodb-to-elasticsearch",
|
|
3
|
-
"version": "5.41.
|
|
3
|
+
"version": "5.41.2-beta.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -12,10 +12,10 @@
|
|
|
12
12
|
"author": "Webiny Ltd.",
|
|
13
13
|
"dependencies": {
|
|
14
14
|
"@babel/runtime": "7.24.1",
|
|
15
|
-
"@webiny/api-elasticsearch": "5.41.
|
|
16
|
-
"@webiny/aws-sdk": "5.41.
|
|
17
|
-
"@webiny/error": "5.41.
|
|
18
|
-
"@webiny/handler-aws": "5.41.
|
|
15
|
+
"@webiny/api-elasticsearch": "5.41.2-beta.0",
|
|
16
|
+
"@webiny/aws-sdk": "5.41.2-beta.0",
|
|
17
|
+
"@webiny/error": "5.41.2-beta.0",
|
|
18
|
+
"@webiny/handler-aws": "5.41.2-beta.0",
|
|
19
19
|
"p-retry": "4.6.2"
|
|
20
20
|
},
|
|
21
21
|
"devDependencies": {
|
|
@@ -26,9 +26,9 @@
|
|
|
26
26
|
"@babel/preset-env": "7.24.3",
|
|
27
27
|
"@babel/preset-typescript": "7.24.1",
|
|
28
28
|
"@types/aws-lambda": "8.10.136",
|
|
29
|
-
"@webiny/cli": "5.41.
|
|
30
|
-
"@webiny/plugins": "5.41.
|
|
31
|
-
"@webiny/project-utils": "5.41.
|
|
29
|
+
"@webiny/cli": "5.41.2-beta.0",
|
|
30
|
+
"@webiny/plugins": "5.41.2-beta.0",
|
|
31
|
+
"@webiny/project-utils": "5.41.2-beta.0",
|
|
32
32
|
"typescript": "4.9.5"
|
|
33
33
|
},
|
|
34
34
|
"publishConfig": {
|
|
@@ -44,5 +44,5 @@
|
|
|
44
44
|
"__tests__"
|
|
45
45
|
]
|
|
46
46
|
},
|
|
47
|
-
"gitHead": "
|
|
47
|
+
"gitHead": "a259e2af938ceb1e2d35b2bef9edcee4459352de"
|
|
48
48
|
}
|
package/types.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { GenericRecord } from "@webiny/cli/types";
|
|
2
2
|
import { DynamoDBRecord } from "@webiny/handler-aws/types";
|
|
3
|
+
import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
|
|
4
|
+
export type Context = Pick<ElasticsearchContext, "elasticsearch" | "plugins">;
|
|
3
5
|
export interface IOperationsBuilderBuildParams {
|
|
4
6
|
records: DynamoDBRecord[];
|
|
5
7
|
}
|
|
@@ -19,6 +21,7 @@ export interface IDeleteOperationParams {
|
|
|
19
21
|
export interface IOperations {
|
|
20
22
|
items: GenericRecord[];
|
|
21
23
|
total: number;
|
|
24
|
+
clear(): void;
|
|
22
25
|
insert(params: IInsertOperationParams): void;
|
|
23
26
|
modify(params: IModifyOperationParams): void;
|
|
24
27
|
delete(params: IDeleteOperationParams): void;
|
package/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import { GenericRecord } from \"@webiny/cli/types\";\nimport { DynamoDBRecord } from \"@webiny/handler-aws/types\";\n\nexport interface IOperationsBuilderBuildParams {\n records: DynamoDBRecord[];\n}\n\nexport interface IOperationsBuilder {\n build(params: IOperationsBuilderBuildParams): Promise<IOperations>;\n}\n\nexport interface IInsertOperationParams {\n id: string;\n index: string;\n data: GenericRecord;\n}\n\nexport type IModifyOperationParams = IInsertOperationParams;\n\nexport interface IDeleteOperationParams {\n id: string;\n index: string;\n}\n\nexport interface IOperations {\n items: GenericRecord[];\n total: number;\n insert(params: IInsertOperationParams): void;\n modify(params: IModifyOperationParams): void;\n delete(params: IDeleteOperationParams): void;\n}\n\nexport interface IDecompressor {\n decompress(data: GenericRecord): Promise<GenericRecord | null>;\n}\n"],"mappings":"","ignoreList":[]}
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import { GenericRecord } from \"@webiny/cli/types\";\nimport { DynamoDBRecord } from \"@webiny/handler-aws/types\";\nimport { ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\n\nexport type Context = Pick<ElasticsearchContext, \"elasticsearch\" | \"plugins\">;\n\nexport interface IOperationsBuilderBuildParams {\n records: DynamoDBRecord[];\n}\n\nexport interface IOperationsBuilder {\n build(params: IOperationsBuilderBuildParams): Promise<IOperations>;\n}\n\nexport interface IInsertOperationParams {\n id: string;\n index: string;\n data: GenericRecord;\n}\n\nexport type IModifyOperationParams = IInsertOperationParams;\n\nexport interface IDeleteOperationParams {\n id: string;\n index: string;\n}\n\nexport interface IOperations {\n items: GenericRecord[];\n total: number;\n clear(): void;\n insert(params: IInsertOperationParams): void;\n modify(params: IModifyOperationParams): void;\n delete(params: IDeleteOperationParams): void;\n}\n\nexport interface IDecompressor {\n decompress(data: GenericRecord): Promise<GenericRecord | null>;\n}\n"],"mappings":"","ignoreList":[]}
|