@webiny/api-dynamodb-to-elasticsearch 6.4.12-beta.5 → 6.5.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/OperationsBuilder.js +3 -3
- package/OperationsBuilder.js.map +1 -1
- package/SynchronizationBuilder.d.ts +5 -4
- package/SynchronizationBuilder.js +2 -2
- package/SynchronizationBuilder.js.map +1 -1
- package/eventHandler.js +7 -2
- package/eventHandler.js.map +1 -1
- package/execute.d.ts +3 -2
- package/execute.js +10 -9
- package/execute.js.map +1 -1
- package/executeWithRetry.js +1 -1
- package/executeWithRetry.js.map +1 -1
- package/helpers/getNumberEnvVariable.js +1 -1
- package/helpers/getNumberEnvVariable.js.map +1 -1
- package/package.json +11 -11
- package/types.d.ts +1 -2
package/OperationsBuilder.js
CHANGED
|
@@ -8,7 +8,7 @@ class OperationsBuilder {
|
|
|
8
8
|
const operations = new Operations();
|
|
9
9
|
for (const record of params.records){
|
|
10
10
|
if (!record.dynamodb) continue;
|
|
11
|
-
|
|
11
|
+
if (!record.eventName) {
|
|
12
12
|
console.error(`Could not get operation from the record, skipping event "${record.eventID}".`);
|
|
13
13
|
continue;
|
|
14
14
|
}
|
|
@@ -21,8 +21,8 @@ class OperationsBuilder {
|
|
|
21
21
|
if (record.eventName === OperationType.INSERT || record.eventName === OperationType.MODIFY) {
|
|
22
22
|
const newImage = unmarshall(record.dynamodb.NewImage);
|
|
23
23
|
if (!newImage || "object" != typeof newImage || 0 === Object.keys(newImage).length) continue;
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
if (true === newImage.ignore) continue;
|
|
25
|
+
if (!newImage.index) {
|
|
26
26
|
console.error(`Could not get index from the new image, skipping event "${record.eventID}".`);
|
|
27
27
|
console.log({
|
|
28
28
|
newImage
|
package/OperationsBuilder.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OperationsBuilder.js","sources":["../src/OperationsBuilder.ts"],"sourcesContent":["import type { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from \"./types.js\";\nimport { Operations, OperationType } from \"~/Operations.js\";\nimport { unmarshall } from \"~/marshall.js\";\nimport type { CompressionHandler } from \"@webiny/utils/exports/api.js\";\n\ninterface RecordDynamoDbImage {\n data: {\n compression: string;\n value: string;\n };\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n compressor: CompressionHandler.Interface;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n private readonly compressor: CompressionHandler.Interface;\n\n public constructor(params: IOperationsBuilderParams) {\n this.compressor = params.compressor;\n }\n\n public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n const operations = new Operations();\n for (const record of params.records) {\n if (!record.dynamodb) {\n continue;\n } else if (!record.eventName) {\n console.error(\n `Could not get operation from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n if (!keys?.PK || !keys.SK) {\n console.error(\n `Could not get keys from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const id = `${keys.PK}:${keys.SK}`;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n if (\n record.eventName === OperationType.INSERT ||\n record.eventName === OperationType.MODIFY\n ) {\n const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n /**\n * If there is no newImage, silently continue to the next operation.\n */\n if (\n !newImage ||\n typeof newImage !== \"object\" ||\n Object.keys(newImage).length === 0\n ) {\n continue;\n }\n /**\n * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n * need to process it.\n */\n //\n else if (newImage.ignore === true) {\n // Nothing to log here, we are skipping the record intentionally.\n continue;\n }\n /**\n * Also, possibly there is no index?\n */\n //\n else if (!newImage.index) {\n console.error(\n `Could not get index from the new image, skipping event \"${record.eventID}\".`\n );\n console.log({ newImage });\n continue;\n }\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n const data = await this.compressor.decompress(newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.error(\n `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n );\n continue;\n }\n\n operations.insert({\n id,\n index: newImage.index,\n data\n });\n } else if (record.eventName === OperationType.REMOVE) {\n const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n /**\n * If there is no index found, silently continue to the next operation.\n */\n if (!oldImage?.index) {\n continue;\n }\n operations.delete({\n id,\n index: oldImage.index\n });\n }\n }\n return operations;\n }\n}\n"],"names":["OperationsBuilder","params","operations","Operations","record","console","keys","unmarshall","id","OperationType","newImage","Object","data","oldImage"],"mappings":";;AAuBO,MAAMA;IAGT,YAAmBC,MAAgC,CAAE;QACjD,IAAI,CAAC,UAAU,GAAGA,OAAO,UAAU;IACvC;IAEA,MAAa,MAAMA,MAAqC,EAAwB;QAC5E,MAAMC,aAAa,IAAIC;QACvB,KAAK,MAAMC,UAAUH,OAAO,OAAO,CAAE;YACjC,IAAI,CAACG,OAAO,QAAQ,EAChB;
|
|
1
|
+
{"version":3,"file":"OperationsBuilder.js","sources":["../src/OperationsBuilder.ts"],"sourcesContent":["import type { IOperations, IOperationsBuilder, IOperationsBuilderBuildParams } from \"./types.js\";\nimport { Operations, OperationType } from \"~/Operations.js\";\nimport { unmarshall } from \"~/marshall.js\";\nimport type { CompressionHandler } from \"@webiny/utils/exports/api.js\";\n\ninterface RecordDynamoDbImage {\n data: {\n compression: string;\n value: string;\n };\n ignore?: boolean;\n index: string;\n}\n\ninterface RecordDynamoDbKeys {\n PK: string;\n SK: string;\n}\n\nexport interface IOperationsBuilderParams {\n compressor: CompressionHandler.Interface;\n}\n\nexport class OperationsBuilder implements IOperationsBuilder {\n private readonly compressor: CompressionHandler.Interface;\n\n public constructor(params: IOperationsBuilderParams) {\n this.compressor = params.compressor;\n }\n\n public async build(params: IOperationsBuilderBuildParams): Promise<IOperations> {\n const operations = new Operations();\n for (const record of params.records) {\n if (!record.dynamodb) {\n continue;\n } else if (!record.eventName) {\n console.error(\n `Could not get operation from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const keys = unmarshall<RecordDynamoDbKeys>(record.dynamodb.Keys);\n if (!keys?.PK || !keys.SK) {\n console.error(\n `Could not get keys from the record, skipping event \"${record.eventID}\".`\n );\n continue;\n }\n\n const id = `${keys.PK}:${keys.SK}`;\n\n /**\n * On operations other than REMOVE we decompress the data and store it into the Elasticsearch.\n * No need to try to decompress if operation is REMOVE since there is no data sent into that operation.\n */\n if (\n record.eventName === OperationType.INSERT ||\n record.eventName === OperationType.MODIFY\n ) {\n const newImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.NewImage);\n /**\n * If there is no newImage, silently continue to the next operation.\n */\n if (\n !newImage ||\n typeof newImage !== \"object\" ||\n Object.keys(newImage).length === 0\n ) {\n continue;\n }\n /**\n * Note that with the `REMOVE` event, there is no `NewImage` property. Which means,\n * if the `newImage` is `undefined`, we are dealing with a `REMOVE` event and we still\n * need to process it.\n */\n //\n else if (newImage.ignore === true) {\n // Nothing to log here, we are skipping the record intentionally.\n continue;\n }\n /**\n * Also, possibly there is no index?\n */\n //\n else if (!newImage.index) {\n console.error(\n `Could not get index from the new image, skipping event \"${record.eventID}\".`\n );\n console.log({ newImage });\n continue;\n }\n /**\n * We must decompress the data that is going into the Elasticsearch.\n */\n const data = await this.compressor.decompress(newImage.data);\n /**\n * No point in writing null or undefined data into the Elasticsearch.\n * This might happen on some error while decompressing. We will log it.\n *\n * Data should NEVER be null or undefined in the Elasticsearch DynamoDB table, unless it is a delete operations.\n * If it is - it is a bug.\n */\n if (data === undefined || data === null) {\n console.error(\n `Could not get decompressed data, skipping ES operation \"${record.eventName}\", ID ${id}. Skipping...`\n );\n continue;\n }\n\n operations.insert({\n id,\n index: newImage.index,\n data\n });\n } else if (record.eventName === OperationType.REMOVE) {\n const oldImage = unmarshall<RecordDynamoDbImage>(record.dynamodb.OldImage);\n /**\n * If there is no index found, silently continue to the next operation.\n */\n if (!oldImage?.index) {\n continue;\n }\n operations.delete({\n id,\n index: oldImage.index\n });\n }\n }\n return operations;\n }\n}\n"],"names":["OperationsBuilder","params","operations","Operations","record","console","keys","unmarshall","id","OperationType","newImage","Object","data","oldImage"],"mappings":";;AAuBO,MAAMA;IAGT,YAAmBC,MAAgC,CAAE;QACjD,IAAI,CAAC,UAAU,GAAGA,OAAO,UAAU;IACvC;IAEA,MAAa,MAAMA,MAAqC,EAAwB;QAC5E,MAAMC,aAAa,IAAIC;QACvB,KAAK,MAAMC,UAAUH,OAAO,OAAO,CAAE;YACjC,IAAI,CAACG,OAAO,QAAQ,EAChB;YACG,IAAI,CAACA,OAAO,SAAS,EAAE;gBAC1BC,QAAQ,KAAK,CACT,CAAC,yDAAyD,EAAED,OAAO,OAAO,CAAC,EAAE,CAAC;gBAElF;YACJ;YAEA,MAAME,OAAOC,WAA+BH,OAAO,QAAQ,CAAC,IAAI;YAChE,IAAI,CAACE,MAAM,MAAM,CAACA,KAAK,EAAE,EAAE;gBACvBD,QAAQ,KAAK,CACT,CAAC,oDAAoD,EAAED,OAAO,OAAO,CAAC,EAAE,CAAC;gBAE7E;YACJ;YAEA,MAAMI,KAAK,GAAGF,KAAK,EAAE,CAAC,CAAC,EAAEA,KAAK,EAAE,EAAE;YAMlC,IACIF,OAAO,SAAS,KAAKK,cAAc,MAAM,IACzCL,OAAO,SAAS,KAAKK,cAAc,MAAM,EAC3C;gBACE,MAAMC,WAAWH,WAAgCH,OAAO,QAAQ,CAAC,QAAQ;gBAIzE,IACI,CAACM,YACD,AAAoB,YAApB,OAAOA,YACPC,AAAiC,MAAjCA,OAAO,IAAI,CAACD,UAAU,MAAM,EAE5B;gBAQC,IAAIA,AAAoB,SAApBA,SAAS,MAAM,EAEpB;gBAMC,IAAI,CAACA,SAAS,KAAK,EAAE;oBACtBL,QAAQ,KAAK,CACT,CAAC,wDAAwD,EAAED,OAAO,OAAO,CAAC,EAAE,CAAC;oBAEjFC,QAAQ,GAAG,CAAC;wBAAEK;oBAAS;oBACvB;gBACJ;gBAIA,MAAME,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAACF,SAAS,IAAI;gBAQ3D,IAAIE,QAAAA,MAAqC;oBACrCP,QAAQ,KAAK,CACT,CAAC,wDAAwD,EAAED,OAAO,SAAS,CAAC,MAAM,EAAEI,GAAG,aAAa,CAAC;oBAEzG;gBACJ;gBAEAN,WAAW,MAAM,CAAC;oBACdM;oBACA,OAAOE,SAAS,KAAK;oBACrBE;gBACJ;YACJ,OAAO,IAAIR,OAAO,SAAS,KAAKK,cAAc,MAAM,EAAE;gBAClD,MAAMI,WAAWN,WAAgCH,OAAO,QAAQ,CAAC,QAAQ;gBAIzE,IAAI,CAACS,UAAU,OACX;gBAEJX,WAAW,MAAM,CAAC;oBACdM;oBACA,OAAOK,SAAS,KAAK;gBACzB;YACJ;QACJ;QACA,OAAOX;IACX;AACJ"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { IDeleteOperationParams, IInsertOperationParams, IModifyOperationParams } from "./types.js";
|
|
2
2
|
import type { IExecuteWithRetryParams } from "./executeWithRetry.js";
|
|
3
3
|
import type { ITimer } from "@webiny/handler-aws";
|
|
4
|
-
|
|
4
|
+
import { OpenSearchClient } from "@webiny/api-opensearch/exports/api/opensearch.js";
|
|
5
|
+
export type ISynchronizationBuilderExecuteWithRetryParams = Omit<IExecuteWithRetryParams, "opensearch" | "timer" | "maxRunningTime" | "operations">;
|
|
5
6
|
export interface ISynchronizationBuilder {
|
|
6
7
|
insert(params: IInsertOperationParams): void;
|
|
7
8
|
delete(params: IDeleteOperationParams): void;
|
|
@@ -9,11 +10,11 @@ export interface ISynchronizationBuilder {
|
|
|
9
10
|
}
|
|
10
11
|
export interface ISynchronizationBuilderParams {
|
|
11
12
|
timer: ITimer;
|
|
12
|
-
|
|
13
|
+
openSearchClient: OpenSearchClient.Client;
|
|
13
14
|
}
|
|
14
15
|
export declare class SynchronizationBuilder implements ISynchronizationBuilder {
|
|
15
16
|
private readonly timer;
|
|
16
|
-
private readonly
|
|
17
|
+
private readonly openSearchClient;
|
|
17
18
|
private readonly operations;
|
|
18
19
|
constructor(params: ISynchronizationBuilderParams);
|
|
19
20
|
insert(params: IInsertOperationParams): void;
|
|
@@ -3,7 +3,7 @@ import { executeWithRetry } from "./executeWithRetry.js";
|
|
|
3
3
|
class SynchronizationBuilder {
|
|
4
4
|
constructor(params){
|
|
5
5
|
this.timer = params.timer;
|
|
6
|
-
this.
|
|
6
|
+
this.openSearchClient = params.openSearchClient;
|
|
7
7
|
this.operations = new Operations();
|
|
8
8
|
}
|
|
9
9
|
insert(params) {
|
|
@@ -22,7 +22,7 @@ class SynchronizationBuilder {
|
|
|
22
22
|
...params,
|
|
23
23
|
maxRunningTime: this.timer.getRemainingMilliseconds(),
|
|
24
24
|
timer: this.timer,
|
|
25
|
-
|
|
25
|
+
openSearchClient: this.openSearchClient,
|
|
26
26
|
operations: this.operations
|
|
27
27
|
});
|
|
28
28
|
this.operations.clear();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SynchronizationBuilder.js","sources":["../src/SynchronizationBuilder.ts"],"sourcesContent":["import type {\n
|
|
1
|
+
{"version":3,"file":"SynchronizationBuilder.js","sources":["../src/SynchronizationBuilder.ts"],"sourcesContent":["import type {\n IDeleteOperationParams,\n IInsertOperationParams,\n IModifyOperationParams\n} from \"~/types.js\";\nimport { Operations } from \"~/Operations.js\";\nimport type { IExecuteWithRetryParams } from \"~/executeWithRetry.js\";\nimport { executeWithRetry } from \"~/executeWithRetry.js\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport { OpenSearchClient } from \"@webiny/api-opensearch/exports/api/opensearch.js\";\n\nexport type ISynchronizationBuilderExecuteWithRetryParams = Omit<\n IExecuteWithRetryParams,\n \"opensearch\" | \"timer\" | \"maxRunningTime\" | \"operations\"\n>;\n\nexport interface ISynchronizationBuilder {\n insert(params: IInsertOperationParams): void;\n delete(params: IDeleteOperationParams): void;\n build: () => (params?: ISynchronizationBuilderExecuteWithRetryParams) => Promise<void>;\n}\n\nexport interface ISynchronizationBuilderParams {\n timer: ITimer;\n openSearchClient: OpenSearchClient.Client;\n}\n\nexport class SynchronizationBuilder implements ISynchronizationBuilder {\n private readonly timer;\n private readonly openSearchClient;\n private readonly operations;\n\n public constructor(params: ISynchronizationBuilderParams) {\n this.timer = params.timer;\n this.openSearchClient = params.openSearchClient;\n this.operations = new Operations();\n }\n\n public insert(params: IInsertOperationParams): void {\n return this.operations.insert(params);\n }\n\n public modify(params: IModifyOperationParams): void {\n return this.operations.modify(params);\n }\n\n public delete(params: IDeleteOperationParams): void {\n return this.operations.delete(params);\n }\n\n public build() {\n return async (params?: ISynchronizationBuilderExecuteWithRetryParams) => {\n if (this.operations.total === 0) {\n return;\n }\n await executeWithRetry({\n ...params,\n maxRunningTime: this.timer.getRemainingMilliseconds(),\n timer: this.timer,\n openSearchClient: this.openSearchClient,\n operations: this.operations\n });\n this.operations.clear();\n };\n }\n}\n\nexport const createSynchronizationBuilder = (\n params: ISynchronizationBuilderParams\n): ISynchronizationBuilder => {\n return new SynchronizationBuilder(params);\n};\n"],"names":["SynchronizationBuilder","params","Operations","executeWithRetry","createSynchronizationBuilder"],"mappings":";;AA2BO,MAAMA;IAKT,YAAmBC,MAAqC,CAAE;QACtD,IAAI,CAAC,KAAK,GAAGA,OAAO,KAAK;QACzB,IAAI,CAAC,gBAAgB,GAAGA,OAAO,gBAAgB;QAC/C,IAAI,CAAC,UAAU,GAAG,IAAIC;IAC1B;IAEO,OAAOD,MAA8B,EAAQ;QAChD,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAACA;IAClC;IAEO,OAAOA,MAA8B,EAAQ;QAChD,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAACA;IAClC;IAEO,OAAOA,MAA8B,EAAQ;QAChD,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAACA;IAClC;IAEO,QAAQ;QACX,OAAO,OAAOA;YACV,IAAI,AAA0B,MAA1B,IAAI,CAAC,UAAU,CAAC,KAAK,EACrB;YAEJ,MAAME,iBAAiB;gBACnB,GAAGF,MAAM;gBACT,gBAAgB,IAAI,CAAC,KAAK,CAAC,wBAAwB;gBACnD,OAAO,IAAI,CAAC,KAAK;gBACjB,kBAAkB,IAAI,CAAC,gBAAgB;gBACvC,YAAY,IAAI,CAAC,UAAU;YAC/B;YACA,IAAI,CAAC,UAAU,CAAC,KAAK;QACzB;IACJ;AACJ;AAEO,MAAMG,+BAA+B,CACxCH,SAEO,IAAID,uBAAuBC"}
|
package/eventHandler.js
CHANGED
|
@@ -2,12 +2,17 @@ import { createDynamoDBEventHandler, timerFactory } from "@webiny/handler-aws";
|
|
|
2
2
|
import { OperationsBuilder } from "./OperationsBuilder.js";
|
|
3
3
|
import { executeWithRetry } from "./executeWithRetry.js";
|
|
4
4
|
import { CompressionHandler } from "@webiny/utils/exports/api.js";
|
|
5
|
+
import { OpenSearchClient } from "@webiny/api-opensearch/exports/api/opensearch.js";
|
|
5
6
|
const MAX_RUNNING_TIME = 900;
|
|
6
7
|
const createEventHandler = ()=>createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext })=>{
|
|
7
8
|
const timer = timerFactory(lambdaContext);
|
|
8
9
|
const context = ctx;
|
|
9
|
-
|
|
10
|
+
let client;
|
|
11
|
+
try {
|
|
12
|
+
client = context.container.resolve(OpenSearchClient);
|
|
13
|
+
} catch (ex) {
|
|
10
14
|
console.error("Missing opensearch definition on context.");
|
|
15
|
+
console.info(ex);
|
|
11
16
|
return null;
|
|
12
17
|
}
|
|
13
18
|
const compressor = context.container.resolve(CompressionHandler);
|
|
@@ -21,7 +26,7 @@ const createEventHandler = ()=>createDynamoDBEventHandler(async ({ event, contex
|
|
|
21
26
|
await executeWithRetry({
|
|
22
27
|
timer,
|
|
23
28
|
maxRunningTime: MAX_RUNNING_TIME,
|
|
24
|
-
|
|
29
|
+
openSearchClient: client.use(),
|
|
25
30
|
operations
|
|
26
31
|
});
|
|
27
32
|
return null;
|
package/eventHandler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eventHandler.js","sources":["../src/eventHandler.ts"],"sourcesContent":["import { createDynamoDBEventHandler, timerFactory } from \"@webiny/handler-aws\";\nimport type { Context } from \"~/types.js\";\nimport { OperationsBuilder } from \"~/OperationsBuilder.js\";\nimport { executeWithRetry } from \"~/executeWithRetry.js\";\nimport { CompressionHandler } from \"@webiny/utils/exports/api.js\";\n\n/**\n * Also, we need to set the maximum running time for the Lambda Function.\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218\n */\nconst MAX_RUNNING_TIME = 900;\n\nexport const createEventHandler = () => {\n return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => {\n const timer = timerFactory(lambdaContext);\n const context = ctx as unknown as Context;\n
|
|
1
|
+
{"version":3,"file":"eventHandler.js","sources":["../src/eventHandler.ts"],"sourcesContent":["import { createDynamoDBEventHandler, timerFactory } from \"@webiny/handler-aws\";\nimport type { Context } from \"~/types.js\";\nimport { OperationsBuilder } from \"~/OperationsBuilder.js\";\nimport { executeWithRetry } from \"~/executeWithRetry.js\";\nimport { CompressionHandler } from \"@webiny/utils/exports/api.js\";\nimport { OpenSearchClient } from \"@webiny/api-opensearch/exports/api/opensearch.js\";\n\n/**\n * Also, we need to set the maximum running time for the Lambda Function.\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreOpenSearch.ts#L232\n * https://github.com/webiny/webiny-js/blob/f7352d418da2b5ae0b781376be46785aa7ac6ae0/packages/pulumi-aws/src/apps/core/CoreElasticSearch.ts#L218\n */\nconst MAX_RUNNING_TIME = 900;\n\nexport const createEventHandler = () => {\n return createDynamoDBEventHandler(async ({ event, context: ctx, lambdaContext }) => {\n const timer = timerFactory(lambdaContext);\n const context = ctx as unknown as Context;\n\n let client: OpenSearchClient.Interface;\n try {\n client = context.container.resolve(OpenSearchClient);\n } catch (ex) {\n console.error(\"Missing opensearch definition on context.\");\n console.info(ex);\n return null;\n }\n\n const compressor = context.container.resolve(CompressionHandler);\n\n const builder = new OperationsBuilder({\n compressor\n });\n\n const operations = await builder.build({\n records: event.Records\n });\n /**\n * No need to do anything if there are no operations.\n */\n if (operations.total === 0) {\n return null;\n }\n /**\n * Execute the operations with retry.\n */\n await executeWithRetry({\n timer,\n maxRunningTime: MAX_RUNNING_TIME,\n openSearchClient: client.use(),\n operations\n });\n\n return null;\n });\n};\n"],"names":["MAX_RUNNING_TIME","createEventHandler","createDynamoDBEventHandler","event","ctx","lambdaContext","timer","timerFactory","context","client","OpenSearchClient","ex","console","compressor","CompressionHandler","builder","OperationsBuilder","operations","executeWithRetry"],"mappings":";;;;;AAYA,MAAMA,mBAAmB;AAElB,MAAMC,qBAAqB,IACvBC,2BAA2B,OAAO,EAAEC,KAAK,EAAE,SAASC,GAAG,EAAEC,aAAa,EAAE;QAC3E,MAAMC,QAAQC,aAAaF;QAC3B,MAAMG,UAAUJ;QAEhB,IAAIK;QACJ,IAAI;YACAA,SAASD,QAAQ,SAAS,CAAC,OAAO,CAACE;QACvC,EAAE,OAAOC,IAAI;YACTC,QAAQ,KAAK,CAAC;YACdA,QAAQ,IAAI,CAACD;YACb,OAAO;QACX;QAEA,MAAME,aAAaL,QAAQ,SAAS,CAAC,OAAO,CAACM;QAE7C,MAAMC,UAAU,IAAIC,kBAAkB;YAClCH;QACJ;QAEA,MAAMI,aAAa,MAAMF,QAAQ,KAAK,CAAC;YACnC,SAASZ,MAAM,OAAO;QAC1B;QAIA,IAAIc,AAAqB,MAArBA,WAAW,KAAK,EAChB,OAAO;QAKX,MAAMC,iBAAiB;YACnBZ;YACA,gBAAgBN;YAChB,kBAAkBS,OAAO,GAAG;YAC5BQ;QACJ;QAEA,OAAO;IACX"}
|
package/execute.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ITimer } from "@webiny/handler-aws";
|
|
2
|
-
import type {
|
|
2
|
+
import type { IOperations } from "./types.js";
|
|
3
|
+
import { OpenSearchClient } from "@webiny/api-opensearch/exports/api/opensearch.js";
|
|
3
4
|
export interface BulkOperationsResponseBodyItemIndexError {
|
|
4
5
|
reason?: string;
|
|
5
6
|
}
|
|
@@ -17,7 +18,7 @@ export interface IExecuteParams {
|
|
|
17
18
|
timer: ITimer;
|
|
18
19
|
maxRunningTime: number;
|
|
19
20
|
maxProcessorPercent: number;
|
|
20
|
-
|
|
21
|
+
openSearchClient: OpenSearchClient.Client;
|
|
21
22
|
operations: Pick<IOperations, "items" | "total">;
|
|
22
23
|
}
|
|
23
24
|
export declare const execute: (params: IExecuteParams) => () => Promise<void>;
|
package/execute.js
CHANGED
|
@@ -11,23 +11,24 @@ const checkErrors = (result)=>{
|
|
|
11
11
|
if (!result || !result.body || !result.body.items) return;
|
|
12
12
|
for (const item of result.body.items){
|
|
13
13
|
const err = getError(item);
|
|
14
|
-
if (
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
if (err) {
|
|
15
|
+
if ("index" === err) {
|
|
16
|
+
if ("true" === process.env.DEBUG) console.error("Bulk response", JSON.stringify(result, null, 2));
|
|
17
|
+
continue;
|
|
18
|
+
}
|
|
19
|
+
console.error("Body item with error", item);
|
|
20
|
+
throw new WebinyError(err, "DYNAMODB_TO_OPENSEARCH_ERROR", item);
|
|
18
21
|
}
|
|
19
|
-
console.error("Body item with error", item);
|
|
20
|
-
throw new WebinyError(err, "DYNAMODB_TO_OPENSEARCH_ERROR", item);
|
|
21
22
|
}
|
|
22
23
|
};
|
|
23
24
|
const execute = (params)=>async ()=>{
|
|
24
|
-
const {
|
|
25
|
+
const { openSearchClient, timer, maxRunningTime, maxProcessorPercent, operations } = params;
|
|
25
26
|
if (0 === operations.total) return;
|
|
26
27
|
const remainingTime = timer.getRemainingSeconds();
|
|
27
28
|
const runningTime = maxRunningTime - remainingTime;
|
|
28
29
|
const maxWaitingTime = remainingTime - 90;
|
|
29
30
|
if (shouldShowLogs()) console.debug(`The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`);
|
|
30
|
-
const healthCheck = createWaitUntilHealthy(
|
|
31
|
+
const healthCheck = createWaitUntilHealthy(openSearchClient, {
|
|
31
32
|
minClusterHealthStatus: OpenSearchCatClusterHealthStatus.Yellow,
|
|
32
33
|
waitingTimeStep: 30,
|
|
33
34
|
maxProcessorPercent,
|
|
@@ -75,7 +76,7 @@ const execute = (params)=>async ()=>{
|
|
|
75
76
|
throw ex;
|
|
76
77
|
}
|
|
77
78
|
try {
|
|
78
|
-
const res = await
|
|
79
|
+
const res = await openSearchClient.bulk({
|
|
79
80
|
body: operations.items
|
|
80
81
|
});
|
|
81
82
|
checkErrors(res);
|
package/execute.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execute.js","sources":["../src/execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n OpenSearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-opensearch\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport type { ApiResponse } from \"@webiny/api-opensearch/types.js\";\n\nimport { WebinyError } from \"@webiny/error\";\nimport type {
|
|
1
|
+
{"version":3,"file":"execute.js","sources":["../src/execute.ts"],"sourcesContent":["import {\n createWaitUntilHealthy,\n OpenSearchCatClusterHealthStatus,\n UnhealthyClusterError,\n WaitingHealthyClusterAbortedError\n} from \"@webiny/api-opensearch\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport type { ApiResponse } from \"@webiny/api-opensearch/types.js\";\n\nimport { WebinyError } from \"@webiny/error\";\nimport type { IOperations } from \"./types.js\";\nimport { shouldShowLogs } from \"~/helpers/shouldShowLogs.js\";\nimport { OpenSearchClient } from \"@webiny/api-opensearch/exports/api/opensearch.js\";\n\nexport interface BulkOperationsResponseBodyItemIndexError {\n reason?: string;\n}\n\nexport interface BulkOperationsResponseBodyItemIndex {\n error?: BulkOperationsResponseBodyItemIndexError;\n}\n\nexport interface BulkOperationsResponseBodyItem {\n index?: BulkOperationsResponseBodyItemIndex;\n error?: string;\n}\n\nexport interface BulkOperationsResponseBody {\n items: BulkOperationsResponseBodyItem[];\n}\n\nexport interface IExecuteParams {\n timer: ITimer;\n maxRunningTime: number;\n maxProcessorPercent: number;\n openSearchClient: OpenSearchClient.Client;\n operations: Pick<IOperations, \"items\" | \"total\">;\n}\n\nconst getError = (item: BulkOperationsResponseBodyItem): string | null => {\n if (!item.index?.error?.reason) {\n return null;\n }\n const reason = item.index.error.reason;\n if (reason.match(/no such index \\[([a-zA-Z0-9_-]+)\\]/) !== null) {\n return \"index\";\n }\n return reason;\n};\n\nconst checkErrors = (result?: ApiResponse): void => {\n if (!result || !result.body || !result.body.items) {\n return;\n }\n for (const item of result.body.items) {\n const err = getError(item);\n if (!err) {\n continue;\n } else if (err === \"index\") {\n if (process.env.DEBUG === \"true\") {\n console.error(\"Bulk response\", JSON.stringify(result, null, 2));\n }\n continue;\n }\n console.error(\"Body item with error\", item);\n throw new WebinyError(err, \"DYNAMODB_TO_OPENSEARCH_ERROR\", item);\n }\n};\n\nexport const execute = (params: IExecuteParams) => {\n return async (): Promise<void> => {\n const { openSearchClient, timer, maxRunningTime, maxProcessorPercent, operations } = params;\n\n if (operations.total === 0) {\n return;\n }\n\n const remainingTime = timer.getRemainingSeconds();\n const runningTime = maxRunningTime - remainingTime;\n const maxWaitingTime = remainingTime - 90;\n\n if (shouldShowLogs()) {\n console.debug(\n `The Lambda is already running for ${runningTime}s. Setting Health Check max waiting time: ${maxWaitingTime}s`\n );\n }\n\n const healthCheck = createWaitUntilHealthy(openSearchClient, {\n minClusterHealthStatus: OpenSearchCatClusterHealthStatus.Yellow,\n waitingTimeStep: 30,\n maxProcessorPercent,\n maxWaitingTime\n });\n\n // const log = context.logger.withSource(\"dynamodbToElasticsearch\");\n const log = {\n notice: (...params: any[]) => {\n console.log(...params);\n },\n debug: (...params: any[]) => {\n console.debug(...params);\n },\n info: (...params: any[]) => {\n console.info(...params);\n },\n warn: (...params: any[]) => {\n console.warn(...params);\n },\n error: (...params: any[]) => {\n console.error(...params);\n }\n };\n\n try {\n await healthCheck.wait({\n async onUnhealthy({ startedAt, runs, mustEndAt, waitingTimeStep, waitingReason }) {\n console.debug(`Cluster is unhealthy on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n },\n async onTimeout({ startedAt, runs, waitingTimeStep, mustEndAt, waitingReason }) {\n console.error(`Cluster health check timeout on run #${runs}.`, {\n startedAt,\n mustEndAt,\n waitingTimeStep,\n waitingReason\n });\n }\n });\n } catch (ex) {\n if (\n ex instanceof UnhealthyClusterError ||\n ex instanceof WaitingHealthyClusterAbortedError\n ) {\n throw ex;\n }\n console.error(`Cluster health check failed.`, ex);\n throw ex;\n }\n\n try {\n const res = await openSearchClient.bulk({\n body: operations.items\n });\n checkErrors(res);\n } catch (error) {\n log.error(error, {\n tenant: \"root\"\n });\n\n if (shouldShowLogs() === false) {\n throw error;\n }\n const meta = error?.meta || {};\n delete meta[\"meta\"];\n console.error(\"Bulk error\", JSON.stringify(error, null, 2));\n throw error;\n }\n if (shouldShowLogs() === false) {\n return;\n }\n console.info(`Transferred ${operations.total} record operations to Elasticsearch.`);\n };\n};\n"],"names":["getError","item","reason","checkErrors","result","err","process","console","JSON","WebinyError","execute","params","openSearchClient","timer","maxRunningTime","maxProcessorPercent","operations","remainingTime","runningTime","maxWaitingTime","shouldShowLogs","healthCheck","createWaitUntilHealthy","OpenSearchCatClusterHealthStatus","log","startedAt","runs","mustEndAt","waitingTimeStep","waitingReason","ex","UnhealthyClusterError","WaitingHealthyClusterAbortedError","res","error","meta"],"mappings":";;;AAuCA,MAAMA,WAAW,CAACC;IACd,IAAI,CAACA,KAAK,KAAK,EAAE,OAAO,QACpB,OAAO;IAEX,MAAMC,SAASD,KAAK,KAAK,CAAC,KAAK,CAAC,MAAM;IACtC,IAAIC,AAAuD,SAAvDA,OAAO,KAAK,CAAC,uCACb,OAAO;IAEX,OAAOA;AACX;AAEA,MAAMC,cAAc,CAACC;IACjB,IAAI,CAACA,UAAU,CAACA,OAAO,IAAI,IAAI,CAACA,OAAO,IAAI,CAAC,KAAK,EAC7C;IAEJ,KAAK,MAAMH,QAAQG,OAAO,IAAI,CAAC,KAAK,CAAE;QAClC,MAAMC,MAAML,SAASC;QACrB,IAAKI;YAEE,IAAIA,AAAQ,YAARA,KAAiB;gBACxB,IAAIC,AAAsB,WAAtBA,QAAQ,GAAG,CAAC,KAAK,EACjBC,QAAQ,KAAK,CAAC,iBAAiBC,KAAK,SAAS,CAACJ,QAAQ,MAAM;gBAEhE;YACJ;YACAG,QAAQ,KAAK,CAAC,wBAAwBN;YACtC,MAAM,IAAIQ,YAAYJ,KAAK,gCAAgCJ;;IAC/D;AACJ;AAEO,MAAMS,UAAU,CAACC,SACb;QACH,MAAM,EAAEC,gBAAgB,EAAEC,KAAK,EAAEC,cAAc,EAAEC,mBAAmB,EAAEC,UAAU,EAAE,GAAGL;QAErF,IAAIK,AAAqB,MAArBA,WAAW,KAAK,EAChB;QAGJ,MAAMC,gBAAgBJ,MAAM,mBAAmB;QAC/C,MAAMK,cAAcJ,iBAAiBG;QACrC,MAAME,iBAAiBF,gBAAgB;QAEvC,IAAIG,kBACAb,QAAQ,KAAK,CACT,CAAC,kCAAkC,EAAEW,YAAY,0CAA0C,EAAEC,eAAe,CAAC,CAAC;QAItH,MAAME,cAAcC,uBAAuBV,kBAAkB;YACzD,wBAAwBW,iCAAiC,MAAM;YAC/D,iBAAiB;YACjBR;YACAI;QACJ;QAGA,MAAMK,MAAM;YACR,QAAQ,CAAC,GAAGb;gBACRJ,QAAQ,GAAG,IAAII;YACnB;YACA,OAAO,CAAC,GAAGA;gBACPJ,QAAQ,KAAK,IAAII;YACrB;YACA,MAAM,CAAC,GAAGA;gBACNJ,QAAQ,IAAI,IAAII;YACpB;YACA,MAAM,CAAC,GAAGA;gBACNJ,QAAQ,IAAI,IAAII;YACpB;YACA,OAAO,CAAC,GAAGA;gBACPJ,QAAQ,KAAK,IAAII;YACrB;QACJ;QAEA,IAAI;YACA,MAAMU,YAAY,IAAI,CAAC;gBACnB,MAAM,aAAY,EAAEI,SAAS,EAAEC,IAAI,EAAEC,SAAS,EAAEC,eAAe,EAAEC,aAAa,EAAE;oBAC5EtB,QAAQ,KAAK,CAAC,CAAC,6BAA6B,EAAEmB,KAAK,CAAC,CAAC,EAAE;wBACnDD;wBACAE;wBACAC;wBACAC;oBACJ;gBACJ;gBACA,MAAM,WAAU,EAAEJ,SAAS,EAAEC,IAAI,EAAEE,eAAe,EAAED,SAAS,EAAEE,aAAa,EAAE;oBAC1EtB,QAAQ,KAAK,CAAC,CAAC,qCAAqC,EAAEmB,KAAK,CAAC,CAAC,EAAE;wBAC3DD;wBACAE;wBACAC;wBACAC;oBACJ;gBACJ;YACJ;QACJ,EAAE,OAAOC,IAAI;YACT,IACIA,cAAcC,yBACdD,cAAcE,mCAEd,MAAMF;YAEVvB,QAAQ,KAAK,CAAC,gCAAgCuB;YAC9C,MAAMA;QACV;QAEA,IAAI;YACA,MAAMG,MAAM,MAAMrB,iBAAiB,IAAI,CAAC;gBACpC,MAAMI,WAAW,KAAK;YAC1B;YACAb,YAAY8B;QAChB,EAAE,OAAOC,OAAO;YACZV,IAAI,KAAK,CAACU,OAAO;gBACb,QAAQ;YACZ;YAEA,IAAId,AAAqB,UAArBA,kBACA,MAAMc;YAEV,MAAMC,OAAOD,OAAO,QAAQ,CAAC;YAC7B,OAAOC,IAAI,CAAC,OAAO;YACnB5B,QAAQ,KAAK,CAAC,cAAcC,KAAK,SAAS,CAAC0B,OAAO,MAAM;YACxD,MAAMA;QACV;QACA,IAAId,AAAqB,UAArBA,kBACA;QAEJb,QAAQ,IAAI,CAAC,CAAC,YAAY,EAAES,WAAW,KAAK,CAAC,oCAAoC,CAAC;IACtF"}
|
package/executeWithRetry.js
CHANGED
|
@@ -14,7 +14,7 @@ const executeWithRetry = async (params)=>{
|
|
|
14
14
|
timer: params.timer,
|
|
15
15
|
maxRunningTime: params.maxRunningTime,
|
|
16
16
|
maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,
|
|
17
|
-
|
|
17
|
+
openSearchClient: params.openSearchClient,
|
|
18
18
|
operations: params.operations
|
|
19
19
|
}), {
|
|
20
20
|
maxRetryTime,
|
package/executeWithRetry.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executeWithRetry.js","sources":["../src/executeWithRetry.ts"],"sourcesContent":["import type { IExecuteParams } from \"~/execute.js\";\nimport { execute } from \"~/execute.js\";\nimport { NotEnoughRemainingTimeError } from \"~/NotEnoughRemainingTimeError.js\";\nimport pRetry from \"p-retry\";\nimport { getNumberEnvVariable } from \"./helpers/getNumberEnvVariable.js\";\n\nconst minRemainingSecondsToTimeout = 120;\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n \"MAX_ES_PROCESSOR\",\n process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\nexport interface IExecuteWithRetryParams extends Omit<IExecuteParams, \"maxProcessorPercent\"> {\n maxRetryTime?: number;\n retries?: number;\n minTimeout?: number;\n maxTimeout?: number;\n maxProcessorPercent?: number;\n}\n\nexport const executeWithRetry = async (params: IExecuteWithRetryParams) => {\n const maxRetryTime = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_OPENSEARCH_MAX_RETRY_TIME\",\n params.maxRetryTime || 300000\n );\n const retries = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_OPENSEARCH_RETRIES\",\n params.retries || 20\n );\n const minTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_OPENSEARCH_MIN_TIMEOUT\",\n params.minTimeout || 1500\n );\n const maxTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_OPENSEARCH_MAX_TIMEOUT\",\n params.maxTimeout || 30000\n );\n\n try {\n await pRetry(\n execute({\n timer: params.timer,\n maxRunningTime: params.maxRunningTime,\n maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,\n
|
|
1
|
+
{"version":3,"file":"executeWithRetry.js","sources":["../src/executeWithRetry.ts"],"sourcesContent":["import type { IExecuteParams } from \"~/execute.js\";\nimport { execute } from \"~/execute.js\";\nimport { NotEnoughRemainingTimeError } from \"~/NotEnoughRemainingTimeError.js\";\nimport pRetry from \"p-retry\";\nimport { getNumberEnvVariable } from \"./helpers/getNumberEnvVariable.js\";\n\nconst minRemainingSecondsToTimeout = 120;\n\nconst MAX_PROCESSOR_PERCENT = getNumberEnvVariable(\n \"MAX_ES_PROCESSOR\",\n process.env.NODE_ENV === \"test\" ? 101 : 98\n);\n\nexport interface IExecuteWithRetryParams extends Omit<IExecuteParams, \"maxProcessorPercent\"> {\n maxRetryTime?: number;\n retries?: number;\n minTimeout?: number;\n maxTimeout?: number;\n maxProcessorPercent?: number;\n}\n\nexport const executeWithRetry = async (params: IExecuteWithRetryParams) => {\n const maxRetryTime = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_OPENSEARCH_MAX_RETRY_TIME\",\n params.maxRetryTime || 300000\n );\n const retries = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_OPENSEARCH_RETRIES\",\n params.retries || 20\n );\n const minTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_OPENSEARCH_MIN_TIMEOUT\",\n params.minTimeout || 1500\n );\n const maxTimeout = getNumberEnvVariable(\n \"WEBINY_DYNAMODB_TO_OPENSEARCH_MAX_TIMEOUT\",\n params.maxTimeout || 30000\n );\n\n try {\n await pRetry(\n execute({\n timer: params.timer,\n maxRunningTime: params.maxRunningTime,\n maxProcessorPercent: params.maxProcessorPercent || MAX_PROCESSOR_PERCENT,\n openSearchClient: params.openSearchClient,\n operations: params.operations\n }),\n {\n maxRetryTime,\n retries,\n minTimeout,\n maxTimeout,\n onFailedAttempt: ({ error, attemptNumber }) => {\n if (params.timer.getRemainingSeconds() < minRemainingSecondsToTimeout) {\n throw new NotEnoughRemainingTimeError(error);\n }\n /**\n * We will only log attempts which are after 3/4 of total attempts.\n */\n if (attemptNumber < retries * 0.75) {\n return;\n }\n console.error(`Attempt #${attemptNumber} failed.`);\n console.error(error);\n }\n }\n );\n } catch (ex) {\n // TODO implement storing of failed operations\n throw ex;\n }\n};\n"],"names":["minRemainingSecondsToTimeout","MAX_PROCESSOR_PERCENT","getNumberEnvVariable","process","executeWithRetry","params","maxRetryTime","retries","minTimeout","maxTimeout","pRetry","execute","error","attemptNumber","NotEnoughRemainingTimeError","console","ex"],"mappings":";;;;AAMA,MAAMA,+BAA+B;AAErC,MAAMC,wBAAwBC,qBAC1B,oBACAC,AAAyB,WAAzBA,QAAQ,GAAG,CAAC,QAAQ,GAAc,MAAM;AAWrC,MAAMC,mBAAmB,OAAOC;IACnC,MAAMC,eAAeJ,qBACjB,gDACAG,OAAO,YAAY,IAAI;IAE3B,MAAME,UAAUL,qBACZ,yCACAG,OAAO,OAAO,IAAI;IAEtB,MAAMG,aAAaN,qBACf,6CACAG,OAAO,UAAU,IAAI;IAEzB,MAAMI,aAAaP,qBACf,6CACAG,OAAO,UAAU,IAAI;IAGzB,IAAI;QACA,MAAMK,QACFC,QAAQ;YACJ,OAAON,OAAO,KAAK;YACnB,gBAAgBA,OAAO,cAAc;YACrC,qBAAqBA,OAAO,mBAAmB,IAAIJ;YACnD,kBAAkBI,OAAO,gBAAgB;YACzC,YAAYA,OAAO,UAAU;QACjC,IACA;YACIC;YACAC;YACAC;YACAC;YACA,iBAAiB,CAAC,EAAEG,KAAK,EAAEC,aAAa,EAAE;gBACtC,IAAIR,OAAO,KAAK,CAAC,mBAAmB,KAAKL,8BACrC,MAAM,IAAIc,4BAA4BF;gBAK1C,IAAIC,gBAAgBN,AAAU,OAAVA,SAChB;gBAEJQ,QAAQ,KAAK,CAAC,CAAC,SAAS,EAAEF,cAAc,QAAQ,CAAC;gBACjDE,QAAQ,KAAK,CAACH;YAClB;QACJ;IAER,EAAE,OAAOI,IAAI;QAET,MAAMA;IACV;AACJ"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers/getNumberEnvVariable.js","sources":["../../src/helpers/getNumberEnvVariable.ts"],"sourcesContent":["export const getNumberEnvVariable = (name: string, def: number): number => {\n const input = process.env[name];\n const value = Number(input);\n if (isNaN(value)) {\n return def;\n } else if (value <= 0) {\n return def;\n }\n return value;\n};\n"],"names":["getNumberEnvVariable","name","def","input","process","value","Number","isNaN"],"mappings":"AAAO,MAAMA,uBAAuB,CAACC,MAAcC;IAC/C,MAAMC,QAAQC,QAAQ,GAAG,CAACH,KAAK;IAC/B,MAAMI,QAAQC,OAAOH;IACrB,IAAII,MAAMF,QACN,OAAOH;
|
|
1
|
+
{"version":3,"file":"helpers/getNumberEnvVariable.js","sources":["../../src/helpers/getNumberEnvVariable.ts"],"sourcesContent":["export const getNumberEnvVariable = (name: string, def: number): number => {\n const input = process.env[name];\n const value = Number(input);\n if (isNaN(value)) {\n return def;\n } else if (value <= 0) {\n return def;\n }\n return value;\n};\n"],"names":["getNumberEnvVariable","name","def","input","process","value","Number","isNaN"],"mappings":"AAAO,MAAMA,uBAAuB,CAACC,MAAcC;IAC/C,MAAMC,QAAQC,QAAQ,GAAG,CAACH,KAAK;IAC/B,MAAMI,QAAQC,OAAOH;IACrB,IAAII,MAAMF,QACN,OAAOH;IACJ,IAAIG,SAAS,GAChB,OAAOH;IAEX,OAAOG;AACX"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webiny/api-dynamodb-to-elasticsearch",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.5.0-beta.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./index.js",
|
|
@@ -15,18 +15,18 @@
|
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"author": "Webiny Ltd.",
|
|
17
17
|
"dependencies": {
|
|
18
|
-
"@webiny/api": "6.
|
|
19
|
-
"@webiny/api-opensearch": "6.
|
|
20
|
-
"@webiny/aws-sdk": "6.
|
|
21
|
-
"@webiny/error": "6.
|
|
22
|
-
"@webiny/handler-aws": "6.
|
|
23
|
-
"@webiny/utils": "6.
|
|
24
|
-
"p-retry": "8.0.
|
|
18
|
+
"@webiny/api": "6.5.0-beta.0",
|
|
19
|
+
"@webiny/api-opensearch": "6.5.0-beta.0",
|
|
20
|
+
"@webiny/aws-sdk": "6.5.0-beta.0",
|
|
21
|
+
"@webiny/error": "6.5.0-beta.0",
|
|
22
|
+
"@webiny/handler-aws": "6.5.0-beta.0",
|
|
23
|
+
"@webiny/utils": "6.5.0-beta.0",
|
|
24
|
+
"p-retry": "8.0.0"
|
|
25
25
|
},
|
|
26
26
|
"devDependencies": {
|
|
27
|
-
"@webiny/build-tools": "6.
|
|
28
|
-
"@webiny/plugins": "6.
|
|
29
|
-
"typescript": "
|
|
27
|
+
"@webiny/build-tools": "6.5.0-beta.0",
|
|
28
|
+
"@webiny/plugins": "6.5.0-beta.0",
|
|
29
|
+
"typescript": "7.0.2"
|
|
30
30
|
},
|
|
31
31
|
"publishConfig": {
|
|
32
32
|
"access": "public"
|
package/types.d.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { GenericRecord } from "@webiny/api/types.js";
|
|
2
2
|
import type { DynamoDBRecord, Context as HandlerContext } from "@webiny/handler-aws/types.js";
|
|
3
|
-
import type { OpenSearchContext } from "@webiny/api-opensearch/types.js";
|
|
4
3
|
export interface IOperationsBuilderBuildParams {
|
|
5
4
|
records: DynamoDBRecord[];
|
|
6
5
|
}
|
|
@@ -25,5 +24,5 @@ export interface IOperations {
|
|
|
25
24
|
modify(params: IModifyOperationParams): void;
|
|
26
25
|
delete(params: IDeleteOperationParams): void;
|
|
27
26
|
}
|
|
28
|
-
export interface Context extends
|
|
27
|
+
export interface Context extends HandlerContext {
|
|
29
28
|
}
|