@webiny/api-elasticsearch-tasks 0.0.0-unstable.eb196ccd2f → 0.0.0-unstable.f6dc066313
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/abstractions/DbRegistry.d.ts +11 -0
- package/abstractions/DbRegistry.js +4 -0
- package/abstractions/DbRegistry.js.map +1 -0
- package/abstractions/OpensearchTenantIndexFactory.d.ts +14 -0
- package/abstractions/OpensearchTenantIndexFactory.js +4 -0
- package/abstractions/OpensearchTenantIndexFactory.js.map +1 -0
- package/abstractions/SynchronizationContext.d.ts +6 -0
- package/abstractions/SynchronizationContext.js +4 -0
- package/abstractions/SynchronizationContext.js.map +1 -0
- package/errors/IndexSettingsGetError.d.ts +1 -1
- package/errors/IndexSettingsGetError.js +2 -10
- package/errors/IndexSettingsGetError.js.map +1 -1
- package/errors/IndexSettingsSetError.d.ts +1 -1
- package/errors/IndexSettingsSetError.js +2 -10
- package/errors/IndexSettingsSetError.js.map +1 -1
- package/errors/IndexingDisableError.js +1 -8
- package/errors/IndexingDisableError.js.map +1 -1
- package/errors/IndexingEnableError.js +1 -8
- package/errors/IndexingEnableError.js.map +1 -1
- package/errors/index.d.ts +4 -4
- package/errors/index.js +4 -49
- package/errors/index.js.map +1 -1
- package/helpers/getClients.d.ts +5 -0
- package/helpers/getClients.js +14 -0
- package/helpers/getClients.js.map +1 -0
- package/helpers/scan.d.ts +2 -2
- package/helpers/scan.js +3 -10
- package/helpers/scan.js.map +1 -1
- package/index.d.ts +5 -5
- package/index.js +14 -25
- package/index.js.map +1 -1
- package/package.json +26 -30
- package/settings/DisableIndexing.d.ts +2 -2
- package/settings/DisableIndexing.js +3 -10
- package/settings/DisableIndexing.js.map +1 -1
- package/settings/EnableIndexing.d.ts +2 -2
- package/settings/EnableIndexing.js +3 -10
- package/settings/EnableIndexing.js.map +1 -1
- package/settings/IndexManager.d.ts +2 -2
- package/settings/IndexManager.js +10 -17
- package/settings/IndexManager.js.map +1 -1
- package/settings/IndexSettingsManager.d.ts +1 -1
- package/settings/IndexSettingsManager.js +4 -11
- package/settings/IndexSettingsManager.js.map +1 -1
- package/settings/index.d.ts +1 -1
- package/settings/index.js +1 -16
- package/settings/index.js.map +1 -1
- package/settings/types.d.ts +1 -1
- package/settings/types.js +1 -5
- package/settings/types.js.map +1 -1
- package/tasks/Manager.d.ts +15 -25
- package/tasks/Manager.js +10 -31
- package/tasks/Manager.js.map +1 -1
- package/tasks/createIndexes/CreateIndexesTask.d.ts +23 -0
- package/tasks/createIndexes/CreateIndexesTask.js +53 -0
- package/tasks/createIndexes/CreateIndexesTask.js.map +1 -0
- package/tasks/createIndexes/CreateIndexesTaskRunner.d.ts +14 -8
- package/tasks/createIndexes/CreateIndexesTaskRunner.js +22 -29
- package/tasks/createIndexes/CreateIndexesTaskRunner.js.map +1 -1
- package/tasks/createIndexes/OnBeforeTrigger.d.ts +7 -9
- package/tasks/createIndexes/OnBeforeTrigger.js +11 -28
- package/tasks/createIndexes/OnBeforeTrigger.js.map +1 -1
- package/tasks/createIndexes/createIndex.d.ts +1 -1
- package/tasks/createIndexes/createIndex.js +2 -9
- package/tasks/createIndexes/createIndex.js.map +1 -1
- package/tasks/createIndexes/index.d.ts +2 -3
- package/tasks/createIndexes/index.js +17 -63
- package/tasks/createIndexes/index.js.map +1 -1
- package/tasks/createIndexes/listIndexes.d.ts +4 -12
- package/tasks/createIndexes/listIndexes.js +10 -43
- package/tasks/createIndexes/listIndexes.js.map +1 -1
- package/tasks/createIndexes/types.js +1 -5
- package/tasks/dataSynchronization/DataSynchronizationTask.d.ts +34 -0
- package/tasks/dataSynchronization/DataSynchronizationTask.js +69 -0
- package/tasks/dataSynchronization/DataSynchronizationTask.js.map +1 -0
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.d.ts +6 -3
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js +7 -17
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js.map +1 -1
- package/tasks/dataSynchronization/createFactories.d.ts +1 -1
- package/tasks/dataSynchronization/createFactories.js +3 -10
- package/tasks/dataSynchronization/createFactories.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.d.ts +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js +5 -12
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.d.ts +8 -10
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js +18 -24
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.d.ts +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js +6 -13
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.d.ts +1 -1
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js +1 -5
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js +1 -5
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js +1 -8
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js.map +1 -1
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.d.ts +7 -710
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.js +6 -28
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.js.map +1 -1
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.d.ts +1 -4
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js +2 -19
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js.map +1 -1
- package/tasks/dataSynchronization/entities/getTable.d.ts +3 -3
- package/tasks/dataSynchronization/entities/getTable.js +3 -10
- package/tasks/dataSynchronization/entities/getTable.js.map +1 -1
- package/tasks/dataSynchronization/entities/index.d.ts +3 -3
- package/tasks/dataSynchronization/entities/index.js +3 -38
- package/tasks/dataSynchronization/entities/index.js.map +1 -1
- package/tasks/dataSynchronization/index.d.ts +3 -4
- package/tasks/dataSynchronization/index.js +22 -74
- package/tasks/dataSynchronization/index.js.map +1 -1
- package/tasks/dataSynchronization/types.d.ts +9 -8
- package/tasks/dataSynchronization/types.js +1 -5
- package/tasks/dataSynchronization/types.js.map +1 -1
- package/tasks/enableIndexing/EnableIndexingTaskRunner.d.ts +5 -6
- package/tasks/enableIndexing/EnableIndexingTaskRunner.js +5 -13
- package/tasks/enableIndexing/EnableIndexingTaskRunner.js.map +1 -1
- package/tasks/enableIndexing/index.d.ts +2 -3
- package/tasks/enableIndexing/index.js +40 -46
- package/tasks/enableIndexing/index.js.map +1 -1
- package/tasks/enableIndexing/types.js +1 -5
- package/tasks/index.d.ts +4 -4
- package/tasks/index.js +4 -49
- package/tasks/index.js.map +1 -1
- package/tasks/reindexing/ReindexingTaskRunner.d.ts +4 -5
- package/tasks/reindexing/ReindexingTaskRunner.js +18 -26
- package/tasks/reindexing/ReindexingTaskRunner.js.map +1 -1
- package/tasks/reindexing/index.d.ts +1 -1
- package/tasks/reindexing/index.js +1 -16
- package/tasks/reindexing/index.js.map +1 -1
- package/tasks/reindexing/reindexingTaskDefinition.d.ts +2 -2
- package/tasks/reindexing/reindexingTaskDefinition.js +43 -44
- package/tasks/reindexing/reindexingTaskDefinition.js.map +1 -1
- package/types.d.ts +18 -20
- package/types.js +1 -5
- package/types.js.map +1 -1
- package/definitions/entry.d.ts +0 -11
- package/definitions/entry.js +0 -43
- package/definitions/entry.js.map +0 -1
- package/definitions/index.d.ts +0 -2
- package/definitions/index.js +0 -29
- package/definitions/index.js.map +0 -1
- package/definitions/table.d.ts +0 -7
- package/definitions/table.js +0 -23
- package/definitions/table.js.map +0 -1
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.d.ts +0 -23
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.js +0 -25
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.js.map +0 -1
- package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.d.ts +0 -4
- package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.js +0 -13
- package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.js.map +0 -1
|
@@ -1,16 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.DataSynchronizationTaskRunner = void 0;
|
|
7
|
-
var _ElasticsearchSynchronize = require("./elasticsearch/ElasticsearchSynchronize");
|
|
8
|
-
var _ElasticsearchFetcher = require("./elasticsearch/ElasticsearchFetcher");
|
|
9
|
-
class DataSynchronizationTaskRunner {
|
|
1
|
+
import { ElasticsearchFetcher } from "./elasticsearch/ElasticsearchFetcher.js";
|
|
2
|
+
export class DataSynchronizationTaskRunner {
|
|
10
3
|
constructor(params) {
|
|
11
4
|
this.manager = params.manager;
|
|
12
5
|
this.indexManager = params.indexManager;
|
|
13
6
|
this.factories = params.factories;
|
|
7
|
+
this.elasticsearchSynchronize = params.elasticsearchSynchronize;
|
|
14
8
|
}
|
|
15
9
|
async run(input) {
|
|
16
10
|
this.validateFlow(input);
|
|
@@ -22,24 +16,21 @@ class DataSynchronizationTaskRunner {
|
|
|
22
16
|
const sync = this.factories.elasticsearchToDynamoDb({
|
|
23
17
|
manager: this.manager,
|
|
24
18
|
indexManager: this.indexManager,
|
|
25
|
-
synchronize:
|
|
26
|
-
|
|
27
|
-
timer: this.manager.timer
|
|
28
|
-
}),
|
|
29
|
-
fetcher: new _ElasticsearchFetcher.ElasticsearchFetcher({
|
|
19
|
+
synchronize: this.elasticsearchSynchronize,
|
|
20
|
+
fetcher: new ElasticsearchFetcher({
|
|
30
21
|
client: this.manager.elasticsearch
|
|
31
22
|
})
|
|
32
23
|
});
|
|
33
24
|
try {
|
|
34
25
|
return await sync.run(input);
|
|
35
26
|
} catch (ex) {
|
|
36
|
-
return this.manager.response.error(ex);
|
|
27
|
+
return this.manager.controller.response.error(ex);
|
|
37
28
|
}
|
|
38
29
|
}
|
|
39
30
|
/**
|
|
40
31
|
* We are done.
|
|
41
32
|
*/
|
|
42
|
-
return this.manager.response.done();
|
|
33
|
+
return this.manager.controller.response.done();
|
|
43
34
|
}
|
|
44
35
|
validateFlow(input) {
|
|
45
36
|
if (!input.flow) {
|
|
@@ -50,6 +41,5 @@ class DataSynchronizationTaskRunner {
|
|
|
50
41
|
throw new Error(`Invalid flow "${input.flow}". Allowed flows: ${Object.keys(this.factories).join(", ")}.`);
|
|
51
42
|
}
|
|
52
43
|
}
|
|
53
|
-
exports.DataSynchronizationTaskRunner = DataSynchronizationTaskRunner;
|
|
54
44
|
|
|
55
45
|
//# sourceMappingURL=DataSynchronizationTaskRunner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["ElasticsearchFetcher","DataSynchronizationTaskRunner","constructor","params","manager","indexManager","factories","elasticsearchSynchronize","run","input","validateFlow","flow","elasticsearchToDynamoDb","finished","sync","synchronize","fetcher","client","elasticsearch","ex","controller","response","error","done","Error","Object","keys","join"],"sources":["DataSynchronizationTaskRunner.ts"],"sourcesContent":["import type {\n IDataSynchronizationInput,\n IDataSynchronizationManager,\n IFactories\n} from \"~/tasks/dataSynchronization/types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport { ElasticsearchSynchronize } from \"~/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js\";\nimport { ElasticsearchFetcher } from \"~/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js\";\n\nexport interface IDataSynchronizationTaskRunnerParams {\n manager: IDataSynchronizationManager;\n indexManager: IIndexManager;\n factories: IFactories;\n elasticsearchSynchronize: ElasticsearchSynchronize;\n}\n\nexport class DataSynchronizationTaskRunner {\n private readonly manager: IDataSynchronizationManager;\n private readonly indexManager: IIndexManager;\n private readonly factories: IFactories;\n private readonly elasticsearchSynchronize: ElasticsearchSynchronize;\n\n public constructor(params: IDataSynchronizationTaskRunnerParams) {\n this.manager = params.manager;\n this.indexManager = params.indexManager;\n this.factories = params.factories;\n this.elasticsearchSynchronize = params.elasticsearchSynchronize;\n }\n\n public async run(input: IDataSynchronizationInput) {\n this.validateFlow(input);\n /**\n * Go through the Elasticsearch and delete records which do not exist in the Elasticsearch table.\n */\n //\n if (input.flow === \"elasticsearchToDynamoDb\" && !input.elasticsearchToDynamoDb?.finished) {\n const sync = this.factories.elasticsearchToDynamoDb({\n manager: this.manager,\n indexManager: this.indexManager,\n synchronize: this.elasticsearchSynchronize,\n fetcher: new ElasticsearchFetcher({\n client: this.manager.elasticsearch\n })\n });\n try {\n return await sync.run(input);\n } catch (ex) {\n return this.manager.controller.response.error(ex);\n }\n }\n /**\n * We are done.\n */\n return this.manager.controller.response.done();\n }\n\n private validateFlow(input: IDataSynchronizationInput): void {\n if (!input.flow) {\n throw new Error(`Missing \"flow\" in the input.`);\n } else if (this.factories[input.flow]) {\n return;\n }\n throw new Error(\n `Invalid flow \"${input.flow}\". Allowed flows: ${Object.keys(this.factories).join(\n \", \"\n )}.`\n );\n }\n}\n"],"mappings":"AAOA,SAASA,oBAAoB;AAS7B,OAAO,MAAMC,6BAA6B,CAAC;EAMhCC,WAAWA,CAACC,MAA4C,EAAE;IAC7D,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO;IAC7B,IAAI,CAACC,YAAY,GAAGF,MAAM,CAACE,YAAY;IACvC,IAAI,CAACC,SAAS,GAAGH,MAAM,CAACG,SAAS;IACjC,IAAI,CAACC,wBAAwB,GAAGJ,MAAM,CAACI,wBAAwB;EACnE;EAEA,MAAaC,GAAGA,CAACC,KAAgC,EAAE;IAC/C,IAAI,CAACC,YAAY,CAACD,KAAK,CAAC;IACxB;AACR;AACA;IACQ;IACA,IAAIA,KAAK,CAACE,IAAI,KAAK,yBAAyB,IAAI,CAACF,KAAK,CAACG,uBAAuB,EAAEC,QAAQ,EAAE;MACtF,MAAMC,IAAI,GAAG,IAAI,CAACR,SAAS,CAACM,uBAAuB,CAAC;QAChDR,OAAO,EAAE,IAAI,CAACA,OAAO;QACrBC,YAAY,EAAE,IAAI,CAACA,YAAY;QAC/BU,WAAW,EAAE,IAAI,CAACR,wBAAwB;QAC1CS,OAAO,EAAE,IAAIhB,oBAAoB,CAAC;UAC9BiB,MAAM,EAAE,IAAI,CAACb,OAAO,CAACc;QACzB,CAAC;MACL,CAAC,CAAC;MACF,IAAI;QACA,OAAO,MAAMJ,IAAI,CAACN,GAAG,CAACC,KAAK,CAAC;MAChC,CAAC,CAAC,OAAOU,EAAE,EAAE;QACT,OAAO,IAAI,CAACf,OAAO,CAACgB,UAAU,CAACC,QAAQ,CAACC,KAAK,CAACH,EAAE,CAAC;MACrD;IACJ;IACA;AACR;AACA;IACQ,OAAO,IAAI,CAACf,OAAO,CAACgB,UAAU,CAACC,QAAQ,CAACE,IAAI,CAAC,CAAC;EAClD;EAEQb,YAAYA,CAACD,KAAgC,EAAQ;IACzD,IAAI,CAACA,KAAK,CAACE,IAAI,EAAE;MACb,MAAM,IAAIa,KAAK,CAAC,8BAA8B,CAAC;IACnD,CAAC,MAAM,IAAI,IAAI,CAAClB,SAAS,CAACG,KAAK,CAACE,IAAI,CAAC,EAAE;MACnC;IACJ;IACA,MAAM,IAAIa,KAAK,CACX,iBAAiBf,KAAK,CAACE,IAAI,qBAAqBc,MAAM,CAACC,IAAI,CAAC,IAAI,CAACpB,SAAS,CAAC,CAACqB,IAAI,CAC5E,IACJ,CAAC,GACL,CAAC;EACL;AACJ","ignoreList":[]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import type { IFactories } from "./types";
|
|
1
|
+
import type { IFactories } from "./types.js";
|
|
2
2
|
export declare const createFactories: () => IFactories;
|
|
@@ -1,17 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.createFactories = void 0;
|
|
7
|
-
var _ElasticsearchToDynamoDbSynchronization = require("./elasticsearch/ElasticsearchToDynamoDbSynchronization");
|
|
8
|
-
const createFactories = () => {
|
|
1
|
+
import { ElasticsearchToDynamoDbSynchronization } from "./elasticsearch/ElasticsearchToDynamoDbSynchronization.js";
|
|
2
|
+
export const createFactories = () => {
|
|
9
3
|
return {
|
|
10
4
|
elasticsearchToDynamoDb: params => {
|
|
11
|
-
return new
|
|
5
|
+
return new ElasticsearchToDynamoDbSynchronization(params);
|
|
12
6
|
}
|
|
13
7
|
};
|
|
14
8
|
};
|
|
15
|
-
exports.createFactories = createFactories;
|
|
16
9
|
|
|
17
10
|
//# sourceMappingURL=createFactories.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["ElasticsearchToDynamoDbSynchronization","createFactories","elasticsearchToDynamoDb","params"],"sources":["createFactories.ts"],"sourcesContent":["import type { IFactories } from \"./types.js\";\nimport { ElasticsearchToDynamoDbSynchronization } from \"./elasticsearch/ElasticsearchToDynamoDbSynchronization.js\";\n\nexport const createFactories = (): IFactories => {\n return {\n elasticsearchToDynamoDb: params => {\n return new ElasticsearchToDynamoDbSynchronization(params);\n }\n };\n};\n"],"mappings":"AACA,SAASA,sCAAsC;AAE/C,OAAO,MAAMC,eAAe,GAAGA,CAAA,KAAkB;EAC7C,OAAO;IACHC,uBAAuB,EAAEC,MAAM,IAAI;MAC/B,OAAO,IAAIH,sCAAsC,CAACG,MAAM,CAAC;IAC7D;EACJ,CAAC;AACL,CAAC","ignoreList":[]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Client } from "@webiny/api-elasticsearch";
|
|
2
|
-
import type { IElasticsearchFetcher, IElasticsearchFetcherFetchParams, IElasticsearchFetcherFetchResponse } from "./abstractions/ElasticsearchFetcher";
|
|
2
|
+
import type { IElasticsearchFetcher, IElasticsearchFetcherFetchParams, IElasticsearchFetcherFetchResponse } from "./abstractions/ElasticsearchFetcher.js";
|
|
3
3
|
export interface IElasticsearchFetcherParams {
|
|
4
4
|
client: Client;
|
|
5
5
|
}
|
|
@@ -1,12 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.ElasticsearchFetcher = void 0;
|
|
7
|
-
var _shouldIgnoreEsResponseError = require("./shouldIgnoreEsResponseError");
|
|
8
|
-
var _nodeUtil = require("node:util");
|
|
9
|
-
class ElasticsearchFetcher {
|
|
1
|
+
import { shouldIgnoreEsResponseError } from "./shouldIgnoreEsResponseError.js";
|
|
2
|
+
import { inspect } from "node:util";
|
|
3
|
+
export class ElasticsearchFetcher {
|
|
10
4
|
constructor(params) {
|
|
11
5
|
this.client = params.client;
|
|
12
6
|
}
|
|
@@ -38,9 +32,9 @@ class ElasticsearchFetcher {
|
|
|
38
32
|
/**
|
|
39
33
|
* If we ignore the error, we can continue with the next index.
|
|
40
34
|
*/
|
|
41
|
-
if (
|
|
35
|
+
if (shouldIgnoreEsResponseError(ex)) {
|
|
42
36
|
if (process.env.DEBUG === "true") {
|
|
43
|
-
console.error(
|
|
37
|
+
console.error(inspect(ex, {
|
|
44
38
|
depth: 5,
|
|
45
39
|
showHidden: true
|
|
46
40
|
}));
|
|
@@ -93,6 +87,5 @@ class ElasticsearchFetcher {
|
|
|
93
87
|
};
|
|
94
88
|
}
|
|
95
89
|
}
|
|
96
|
-
exports.ElasticsearchFetcher = ElasticsearchFetcher;
|
|
97
90
|
|
|
98
91
|
//# sourceMappingURL=ElasticsearchFetcher.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["shouldIgnoreEsResponseError","inspect","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","process","env","DEBUG","console","error","depth","showHidden","done","totalCount","items","hits","total","length","undefined","value","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-elasticsearch\";\nimport type {\n IElasticsearchFetcher,\n IElasticsearchFetcherFetchParams,\n IElasticsearchFetcherFetchResponse,\n IElasticsearchFetcherFetchResponseItem\n} from \"./abstractions/ElasticsearchFetcher.js\";\nimport type {\n ElasticsearchSearchResponse,\n PrimitiveValue\n} from \"@webiny/api-elasticsearch/types.js\";\nimport { shouldIgnoreEsResponseError } from \"./shouldIgnoreEsResponseError.js\";\nimport { inspect } from \"node:util\";\n\nexport interface IElasticsearchFetcherParams {\n client: Client;\n}\n\nexport class ElasticsearchFetcher implements IElasticsearchFetcher {\n private readonly client: Client;\n\n public constructor(params: IElasticsearchFetcherParams) {\n this.client = params.client;\n }\n public async fetch({\n index,\n cursor,\n limit\n }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse> {\n let response: ElasticsearchSearchResponse<undefined>;\n try {\n response = await this.client.search({\n index,\n body: {\n query: {\n match_all: {}\n },\n sort: {\n \"id.keyword\": {\n order: \"asc\"\n }\n },\n size: limit + 1,\n track_total_hits: true,\n search_after: cursor,\n _source: false\n }\n });\n } catch (ex) {\n /**\n * If we ignore the error, we can continue with the next index.\n */\n if (shouldIgnoreEsResponseError(ex)) {\n if (process.env.DEBUG === \"true\") {\n console.error(\n inspect(ex, {\n depth: 5,\n showHidden: true\n })\n );\n }\n return {\n done: true,\n totalCount: 0,\n items: []\n };\n }\n console.error(\"Failed to fetch data from Elasticsearch.\", ex);\n throw ex;\n }\n\n const { hits, total } = response.body.hits;\n if (hits.length === 0) {\n return {\n done: true,\n cursor: undefined,\n totalCount: total.value,\n items: []\n };\n }\n\n const hasMoreItems = hits.length > limit;\n let nextCursor: PrimitiveValue[] | undefined;\n if (hasMoreItems) {\n hits.pop();\n nextCursor = hits.at(-1)?.sort;\n }\n const items = hits.reduce<IElasticsearchFetcherFetchResponseItem[]>((collection, hit) => {\n const [PK, SK] = hit._id.split(\":\");\n if (!PK || !SK) {\n return collection;\n }\n collection.push({\n PK,\n SK,\n _id: hit._id,\n index: hit._index\n });\n\n return collection;\n }, []);\n\n return {\n totalCount: total.value,\n cursor: nextCursor,\n done: !nextCursor,\n items\n };\n }\n}\n"],"mappings":"AAWA,SAASA,2BAA2B;AACpC,SAASC,OAAO,QAAQ,WAAW;AAMnC,OAAO,MAAMC,oBAAoB,CAAkC;EAGxDC,WAAWA,CAACC,MAAmC,EAAE;IACpD,IAAI,CAACC,MAAM,GAAGD,MAAM,CAACC,MAAM;EAC/B;EACA,MAAaC,KAAKA,CAAC;IACfC,KAAK;IACLC,MAAM;IACNC;EAC8B,CAAC,EAA+C;IAC9E,IAAIC,QAAgD;IACpD,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACL,MAAM,CAACM,MAAM,CAAC;QAChCJ,KAAK;QACLK,IAAI,EAAE;UACFC,KAAK,EAAE;YACHC,SAAS,EAAE,CAAC;UAChB,CAAC;UACDC,IAAI,EAAE;YACF,YAAY,EAAE;cACVC,KAAK,EAAE;YACX;UACJ,CAAC;UACDC,IAAI,EAAER,KAAK,GAAG,CAAC;UACfS,gBAAgB,EAAE,IAAI;UACtBC,YAAY,EAAEX,MAAM;UACpBY,OAAO,EAAE;QACb;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAIrB,2BAA2B,CAACqB,EAAE,CAAC,EAAE;QACjC,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9BC,OAAO,CAACC,KAAK,CACTzB,OAAO,CAACoB,EAAE,EAAE;YACRM,KAAK,EAAE,CAAC;YACRC,UAAU,EAAE;UAChB,CAAC,CACL,CAAC;QACL;QACA,OAAO;UACHC,IAAI,EAAE,IAAI;UACVC,UAAU,EAAE,CAAC;UACbC,KAAK,EAAE;QACX,CAAC;MACL;MACAN,OAAO,CAACC,KAAK,CAAC,0CAA0C,EAAEL,EAAE,CAAC;MAC7D,MAAMA,EAAE;IACZ;IAEA,MAAM;MAAEW,IAAI;MAAEC;IAAM,CAAC,GAAGvB,QAAQ,CAACE,IAAI,CAACoB,IAAI;IAC1C,IAAIA,IAAI,CAACE,MAAM,KAAK,CAAC,EAAE;MACnB,OAAO;QACHL,IAAI,EAAE,IAAI;QACVrB,MAAM,EAAE2B,SAAS;QACjBL,UAAU,EAAEG,KAAK,CAACG,KAAK;QACvBL,KAAK,EAAE;MACX,CAAC;IACL;IAEA,MAAMM,YAAY,GAAGL,IAAI,CAACE,MAAM,GAAGzB,KAAK;IACxC,IAAI6B,UAAwC;IAC5C,IAAID,YAAY,EAAE;MACdL,IAAI,CAACO,GAAG,CAAC,CAAC;MACVD,UAAU,GAAGN,IAAI,CAACQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAEzB,IAAI;IAClC;IACA,MAAMgB,KAAK,GAAGC,IAAI,CAACS,MAAM,CAA2C,CAACC,UAAU,EAAEC,GAAG,KAAK;MACrF,MAAM,CAACC,EAAE,EAAEC,EAAE,CAAC,GAAGF,GAAG,CAACG,GAAG,CAACC,KAAK,CAAC,GAAG,CAAC;MACnC,IAAI,CAACH,EAAE,IAAI,CAACC,EAAE,EAAE;QACZ,OAAOH,UAAU;MACrB;MACAA,UAAU,CAACM,IAAI,CAAC;QACZJ,EAAE;QACFC,EAAE;QACFC,GAAG,EAAEH,GAAG,CAACG,GAAG;QACZvC,KAAK,EAAEoC,GAAG,CAACM;MACf,CAAC,CAAC;MAEF,OAAOP,UAAU;IACrB,CAAC,EAAE,EAAE,CAAC;IAEN,OAAO;MACHZ,UAAU,EAAEG,KAAK,CAACG,KAAK;MACvB5B,MAAM,EAAE8B,UAAU;MAClBT,IAAI,EAAE,CAACS,UAAU;MACjBP;IACJ,CAAC;EACL;AACJ","ignoreList":[]}
|
|
@@ -1,14 +1,12 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import type {
|
|
3
|
-
import
|
|
4
|
-
|
|
5
|
-
timer: ITimer;
|
|
6
|
-
context: Context;
|
|
7
|
-
}
|
|
1
|
+
import type { TaskController } from "@webiny/api-core/features/task/TaskController/index.js";
|
|
2
|
+
import type { IElasticsearchSynchronize, IElasticsearchSynchronizeExecuteParams, IElasticsearchSynchronizeExecuteResponse } from "./abstractions/ElasticsearchSynchronize.js";
|
|
3
|
+
import { SynchronizationContext } from "../../../abstractions/SynchronizationContext.js";
|
|
4
|
+
import { DbRegistry } from "../../../abstractions/DbRegistry.js";
|
|
8
5
|
export declare class ElasticsearchSynchronize implements IElasticsearchSynchronize {
|
|
9
|
-
private
|
|
10
|
-
private
|
|
11
|
-
|
|
6
|
+
private controller;
|
|
7
|
+
private dbRegistry;
|
|
8
|
+
private context;
|
|
9
|
+
constructor(controller: TaskController.Interface, dbRegistry: DbRegistry.Interface, context: SynchronizationContext.Interface);
|
|
12
10
|
execute(params: IElasticsearchSynchronizeExecuteParams): Promise<IElasticsearchSynchronizeExecuteResponse>;
|
|
13
11
|
private getEntity;
|
|
14
12
|
}
|
|
@@ -1,16 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
var _entities = require("../entities");
|
|
10
|
-
class ElasticsearchSynchronize {
|
|
11
|
-
constructor(params) {
|
|
12
|
-
this.timer = params.timer;
|
|
13
|
-
this.context = params.context;
|
|
1
|
+
import { batchReadAll } from "@webiny/db-dynamodb/utils/batch/batchRead.js";
|
|
2
|
+
import { createSynchronizationBuilder } from "@webiny/api-dynamodb-to-elasticsearch";
|
|
3
|
+
import { getElasticsearchEntity, getElasticsearchEntityType, getTable } from "../entities/index.js";
|
|
4
|
+
export class ElasticsearchSynchronize {
|
|
5
|
+
constructor(controller, dbRegistry, context) {
|
|
6
|
+
this.controller = controller;
|
|
7
|
+
this.dbRegistry = dbRegistry;
|
|
8
|
+
this.context = context;
|
|
14
9
|
}
|
|
15
10
|
async execute(params) {
|
|
16
11
|
const {
|
|
@@ -23,24 +18,24 @@ class ElasticsearchSynchronize {
|
|
|
23
18
|
done: true
|
|
24
19
|
};
|
|
25
20
|
}
|
|
26
|
-
const table =
|
|
21
|
+
const table = getTable({
|
|
27
22
|
type: "es",
|
|
28
|
-
|
|
23
|
+
dbRegistry: this.dbRegistry
|
|
29
24
|
});
|
|
30
25
|
const readableItems = items.map(item => {
|
|
31
26
|
const entity = this.getEntity(item);
|
|
32
|
-
return entity.item.getBatch({
|
|
27
|
+
return entity.item.entity.getBatch({
|
|
33
28
|
PK: item.PK,
|
|
34
29
|
SK: item.SK
|
|
35
30
|
});
|
|
36
31
|
});
|
|
37
|
-
const tableItems = await
|
|
32
|
+
const tableItems = await batchReadAll({
|
|
38
33
|
items: readableItems,
|
|
39
34
|
table
|
|
40
35
|
});
|
|
41
|
-
const elasticsearchSyncBuilder =
|
|
42
|
-
|
|
43
|
-
|
|
36
|
+
const elasticsearchSyncBuilder = createSynchronizationBuilder({
|
|
37
|
+
context: this.context,
|
|
38
|
+
timer: this.controller.runtime
|
|
44
39
|
});
|
|
45
40
|
/**
|
|
46
41
|
* We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table.
|
|
@@ -64,13 +59,12 @@ class ElasticsearchSynchronize {
|
|
|
64
59
|
};
|
|
65
60
|
}
|
|
66
61
|
getEntity(params) {
|
|
67
|
-
const type =
|
|
68
|
-
return
|
|
62
|
+
const type = getElasticsearchEntityType(params);
|
|
63
|
+
return getElasticsearchEntity({
|
|
69
64
|
type,
|
|
70
|
-
|
|
65
|
+
dbRegistry: this.dbRegistry
|
|
71
66
|
});
|
|
72
67
|
}
|
|
73
68
|
}
|
|
74
|
-
exports.ElasticsearchSynchronize = ElasticsearchSynchronize;
|
|
75
69
|
|
|
76
70
|
//# sourceMappingURL=ElasticsearchSynchronize.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["batchReadAll","createSynchronizationBuilder","getElasticsearchEntity","getElasticsearchEntityType","getTable","ElasticsearchSynchronize","constructor","controller","dbRegistry","context","execute","params","items","done","index","length","table","type","readableItems","map","item","entity","getEntity","getBatch","PK","SK","tableItems","elasticsearchSyncBuilder","timer","runtime","exists","some","ddbItem","delete","id","_id","executeWithRetry","build"],"sources":["ElasticsearchSynchronize.ts"],"sourcesContent":["import { batchReadAll } from \"@webiny/db-dynamodb/utils/batch/batchRead.js\";\nimport { createSynchronizationBuilder } from \"@webiny/api-dynamodb-to-elasticsearch\";\nimport type { IGetElasticsearchEntityTypeParams } from \"~/tasks/dataSynchronization/entities/index.js\";\nimport {\n getElasticsearchEntity,\n getElasticsearchEntityType,\n getTable\n} from \"~/tasks/dataSynchronization/entities/index.js\";\nimport type { TaskController } from \"@webiny/api-core/features/task/TaskController/index.js\";\nimport type {\n IElasticsearchSynchronize,\n IElasticsearchSynchronizeExecuteParams,\n IElasticsearchSynchronizeExecuteResponse\n} from \"./abstractions/ElasticsearchSynchronize.js\";\nimport { SynchronizationContext } from \"~/abstractions/SynchronizationContext.js\";\nimport { DbRegistry } from \"~/abstractions/DbRegistry.js\";\n\ninterface IDynamoDbItem {\n PK: string;\n SK: string;\n}\n\nexport class ElasticsearchSynchronize implements IElasticsearchSynchronize {\n public constructor(\n private controller: TaskController.Interface,\n private dbRegistry: DbRegistry.Interface,\n private context: SynchronizationContext.Interface\n ) {}\n\n public async execute(\n params: IElasticsearchSynchronizeExecuteParams\n ): Promise<IElasticsearchSynchronizeExecuteResponse> {\n const { items, done, index } = params;\n if (items.length === 0) {\n return {\n done: true\n };\n }\n\n const table = getTable({\n type: \"es\",\n dbRegistry: this.dbRegistry\n });\n\n const readableItems = items.map(item => {\n const entity = this.getEntity(item);\n return entity.item.entity.getBatch({\n PK: item.PK,\n SK: item.SK\n });\n });\n\n const tableItems = await batchReadAll<IDynamoDbItem>({\n items: readableItems,\n table\n });\n\n const elasticsearchSyncBuilder = createSynchronizationBuilder({\n context: this.context,\n timer: this.controller.runtime\n });\n /**\n * We need to find the items we have in the Elasticsearch but not in the DynamoDB-Elasticsearch table.\n */\n for (const item of items) {\n const exists = tableItems.some(ddbItem => {\n return ddbItem.PK === item.PK && ddbItem.SK === item.SK;\n });\n if (exists) {\n continue;\n }\n elasticsearchSyncBuilder.delete({\n index,\n id: item._id\n });\n }\n\n const executeWithRetry = elasticsearchSyncBuilder.build();\n await executeWithRetry();\n\n return {\n done\n };\n }\n\n private getEntity(\n params: IGetElasticsearchEntityTypeParams\n ): ReturnType<typeof getElasticsearchEntity> {\n const type = getElasticsearchEntityType(params);\n return getElasticsearchEntity({\n type,\n dbRegistry: this.dbRegistry\n });\n }\n}\n"],"mappings":"AAAA,SAASA,YAAY,QAAQ,8CAA8C;AAC3E,SAASC,4BAA4B,QAAQ,uCAAuC;AAEpF,SACIC,sBAAsB,EACtBC,0BAA0B,EAC1BC,QAAQ;AAgBZ,OAAO,MAAMC,wBAAwB,CAAsC;EAChEC,WAAWA,CACNC,UAAoC,EACpCC,UAAgC,EAChCC,OAAyC,EACnD;IAAA,KAHUF,UAAoC,GAApCA,UAAoC;IAAA,KACpCC,UAAgC,GAAhCA,UAAgC;IAAA,KAChCC,OAAyC,GAAzCA,OAAyC;EAClD;EAEH,MAAaC,OAAOA,CAChBC,MAA8C,EACG;IACjD,MAAM;MAAEC,KAAK;MAAEC,IAAI;MAAEC;IAAM,CAAC,GAAGH,MAAM;IACrC,IAAIC,KAAK,CAACG,MAAM,KAAK,CAAC,EAAE;MACpB,OAAO;QACHF,IAAI,EAAE;MACV,CAAC;IACL;IAEA,MAAMG,KAAK,GAAGZ,QAAQ,CAAC;MACnBa,IAAI,EAAE,IAAI;MACVT,UAAU,EAAE,IAAI,CAACA;IACrB,CAAC,CAAC;IAEF,MAAMU,aAAa,GAAGN,KAAK,CAACO,GAAG,CAACC,IAAI,IAAI;MACpC,MAAMC,MAAM,GAAG,IAAI,CAACC,SAAS,CAACF,IAAI,CAAC;MACnC,OAAOC,MAAM,CAACD,IAAI,CAACC,MAAM,CAACE,QAAQ,CAAC;QAC/BC,EAAE,EAAEJ,IAAI,CAACI,EAAE;QACXC,EAAE,EAAEL,IAAI,CAACK;MACb,CAAC,CAAC;IACN,CAAC,CAAC;IAEF,MAAMC,UAAU,GAAG,MAAM1B,YAAY,CAAgB;MACjDY,KAAK,EAAEM,aAAa;MACpBF;IACJ,CAAC,CAAC;IAEF,MAAMW,wBAAwB,GAAG1B,4BAA4B,CAAC;MAC1DQ,OAAO,EAAE,IAAI,CAACA,OAAO;MACrBmB,KAAK,EAAE,IAAI,CAACrB,UAAU,CAACsB;IAC3B,CAAC,CAAC;IACF;AACR;AACA;IACQ,KAAK,MAAMT,IAAI,IAAIR,KAAK,EAAE;MACtB,MAAMkB,MAAM,GAAGJ,UAAU,CAACK,IAAI,CAACC,OAAO,IAAI;QACtC,OAAOA,OAAO,CAACR,EAAE,KAAKJ,IAAI,CAACI,EAAE,IAAIQ,OAAO,CAACP,EAAE,KAAKL,IAAI,CAACK,EAAE;MAC3D,CAAC,CAAC;MACF,IAAIK,MAAM,EAAE;QACR;MACJ;MACAH,wBAAwB,CAACM,MAAM,CAAC;QAC5BnB,KAAK;QACLoB,EAAE,EAAEd,IAAI,CAACe;MACb,CAAC,CAAC;IACN;IAEA,MAAMC,gBAAgB,GAAGT,wBAAwB,CAACU,KAAK,CAAC,CAAC;IACzD,MAAMD,gBAAgB,CAAC,CAAC;IAExB,OAAO;MACHvB;IACJ,CAAC;EACL;EAEQS,SAASA,CACbX,MAAyC,EACA;IACzC,MAAMM,IAAI,GAAGd,0BAA0B,CAACQ,MAAM,CAAC;IAC/C,OAAOT,sBAAsB,CAAC;MAC1Be,IAAI;MACJT,UAAU,EAAE,IAAI,CAACA;IACrB,CAAC,CAAC;EACN;AACJ","ignoreList":[]}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { IDataSynchronizationInput, IElasticsearchSyncParams, ISynchronization, ISynchronizationRunResult } from "../types";
|
|
1
|
+
import type { IDataSynchronizationInput, IElasticsearchSyncParams, ISynchronization, ISynchronizationRunResult } from "../types.js";
|
|
2
2
|
export declare class ElasticsearchToDynamoDbSynchronization implements ISynchronization {
|
|
3
3
|
private readonly manager;
|
|
4
4
|
private readonly indexManager;
|
|
@@ -1,10 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.ElasticsearchToDynamoDbSynchronization = void 0;
|
|
7
|
-
class ElasticsearchToDynamoDbSynchronization {
|
|
1
|
+
export class ElasticsearchToDynamoDbSynchronization {
|
|
8
2
|
constructor(params) {
|
|
9
3
|
this.manager = params.manager;
|
|
10
4
|
this.indexManager = params.indexManager;
|
|
@@ -21,16 +15,16 @@ class ElasticsearchToDynamoDbSynchronization {
|
|
|
21
15
|
}
|
|
22
16
|
let currentIndex = indexes[next];
|
|
23
17
|
while (currentIndex) {
|
|
24
|
-
if (this.manager.isAborted()) {
|
|
25
|
-
return this.manager.response.aborted();
|
|
18
|
+
if (this.manager.controller.runtime.isAborted()) {
|
|
19
|
+
return this.manager.controller.response.aborted();
|
|
26
20
|
}
|
|
27
21
|
/**
|
|
28
22
|
* We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly.
|
|
29
23
|
* We want to leave enough time for possible retries.
|
|
30
24
|
*/
|
|
31
25
|
//
|
|
32
|
-
else if (this.manager.isCloseToTimeout(180)) {
|
|
33
|
-
return this.manager.response.continue({
|
|
26
|
+
else if (this.manager.controller.runtime.isCloseToTimeout(180)) {
|
|
27
|
+
return this.manager.controller.response.continue({
|
|
34
28
|
...input,
|
|
35
29
|
elasticsearchToDynamoDb: {
|
|
36
30
|
...input.elasticsearchToDynamoDb,
|
|
@@ -57,7 +51,7 @@ class ElasticsearchToDynamoDbSynchronization {
|
|
|
57
51
|
const next = indexes.findIndex(index => index === currentIndex) + 1;
|
|
58
52
|
currentIndex = indexes[next];
|
|
59
53
|
}
|
|
60
|
-
return this.manager.response.continue({
|
|
54
|
+
return this.manager.controller.response.continue({
|
|
61
55
|
...input,
|
|
62
56
|
elasticsearchToDynamoDb: {
|
|
63
57
|
finished: true
|
|
@@ -72,6 +66,5 @@ class ElasticsearchToDynamoDbSynchronization {
|
|
|
72
66
|
throw new Error("No Elasticsearch / OpenSearch indexes found.");
|
|
73
67
|
}
|
|
74
68
|
}
|
|
75
|
-
exports.ElasticsearchToDynamoDbSynchronization = ElasticsearchToDynamoDbSynchronization;
|
|
76
69
|
|
|
77
70
|
//# sourceMappingURL=ElasticsearchToDynamoDbSynchronization.js.map
|
package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["ElasticsearchToDynamoDbSynchronization","constructor","params","manager","indexManager","synchronize","fetcher","run","input","lastIndex","elasticsearchToDynamoDb","index","cursor","indexes","fetchAllIndexes","next","findIndex","currentIndex","isAborted","response","aborted","isCloseToTimeout","continue","result","fetch","limit","syncResult","execute","done","items","undefined","finished","list","length","Error"
|
|
1
|
+
{"version":3,"names":["ElasticsearchToDynamoDbSynchronization","constructor","params","manager","indexManager","synchronize","fetcher","run","input","lastIndex","elasticsearchToDynamoDb","index","cursor","indexes","fetchAllIndexes","next","findIndex","currentIndex","controller","runtime","isAborted","response","aborted","isCloseToTimeout","continue","result","fetch","limit","syncResult","execute","done","items","undefined","finished","list","length","Error"],"sources":["ElasticsearchToDynamoDbSynchronization.ts"],"sourcesContent":["import type {\n IDataSynchronizationInput,\n IDataSynchronizationManager,\n IElasticsearchSyncParams,\n ISynchronization,\n ISynchronizationRunResult\n} from \"../types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { NonEmptyArray } from \"@webiny/api/types.js\";\nimport type { IElasticsearchSynchronize } from \"./abstractions/ElasticsearchSynchronize.js\";\nimport type { IElasticsearchFetcher } from \"./abstractions/ElasticsearchFetcher.js\";\n\nexport class ElasticsearchToDynamoDbSynchronization implements ISynchronization {\n private readonly manager: IDataSynchronizationManager;\n private readonly indexManager: IIndexManager;\n private readonly synchronize: IElasticsearchSynchronize;\n private readonly fetcher: IElasticsearchFetcher;\n\n public constructor(params: IElasticsearchSyncParams) {\n this.manager = params.manager;\n this.indexManager = params.indexManager;\n this.synchronize = params.synchronize;\n this.fetcher = params.fetcher;\n }\n\n public async run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult> {\n const lastIndex = input.elasticsearchToDynamoDb?.index;\n let cursor = input.elasticsearchToDynamoDb?.cursor;\n const indexes = await this.fetchAllIndexes();\n\n let next = 0;\n if (lastIndex) {\n next = indexes.findIndex(index => index === lastIndex);\n }\n\n let currentIndex = indexes[next];\n\n while (currentIndex) {\n if (this.manager.controller.runtime.isAborted()) {\n return this.manager.controller.response.aborted();\n }\n /**\n * We will put 180 seconds because we are writing to the Elasticsearch/OpenSearch directly.\n * We want to leave enough time for possible retries.\n */\n //\n else if (this.manager.controller.runtime.isCloseToTimeout(180)) {\n return this.manager.controller.response.continue({\n ...input,\n elasticsearchToDynamoDb: {\n ...input.elasticsearchToDynamoDb,\n index: currentIndex,\n cursor\n }\n });\n }\n\n const result = await this.fetcher.fetch({\n index: currentIndex,\n cursor,\n limit: 100\n });\n\n const syncResult = await this.synchronize.execute({\n done: result.done,\n index: currentIndex,\n items: result.items\n });\n\n if (!syncResult.done && result.cursor) {\n cursor = result.cursor;\n continue;\n }\n cursor = undefined;\n\n const next = indexes.findIndex(index => index === currentIndex) + 1;\n currentIndex = indexes[next];\n }\n\n return this.manager.controller.response.continue({\n ...input,\n elasticsearchToDynamoDb: {\n finished: true\n }\n });\n }\n\n private async fetchAllIndexes(): Promise<NonEmptyArray<string>> {\n const result = await this.indexManager.list();\n if (result.length > 0) {\n return result as NonEmptyArray<string>;\n }\n throw new Error(\"No Elasticsearch / OpenSearch indexes found.\");\n }\n}\n"],"mappings":"AAYA,OAAO,MAAMA,sCAAsC,CAA6B;EAMrEC,WAAWA,CAACC,MAAgC,EAAE;IACjD,IAAI,CAACC,OAAO,GAAGD,MAAM,CAACC,OAAO;IAC7B,IAAI,CAACC,YAAY,GAAGF,MAAM,CAACE,YAAY;IACvC,IAAI,CAACC,WAAW,GAAGH,MAAM,CAACG,WAAW;IACrC,IAAI,CAACC,OAAO,GAAGJ,MAAM,CAACI,OAAO;EACjC;EAEA,MAAaC,GAAGA,CAACC,KAAgC,EAAsC;IACnF,MAAMC,SAAS,GAAGD,KAAK,CAACE,uBAAuB,EAAEC,KAAK;IACtD,IAAIC,MAAM,GAAGJ,KAAK,CAACE,uBAAuB,EAAEE,MAAM;IAClD,MAAMC,OAAO,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAE5C,IAAIC,IAAI,GAAG,CAAC;IACZ,IAAIN,SAAS,EAAE;MACXM,IAAI,GAAGF,OAAO,CAACG,SAAS,CAACL,KAAK,IAAIA,KAAK,KAAKF,SAAS,CAAC;IAC1D;IAEA,IAAIQ,YAAY,GAAGJ,OAAO,CAACE,IAAI,CAAC;IAEhC,OAAOE,YAAY,EAAE;MACjB,IAAI,IAAI,CAACd,OAAO,CAACe,UAAU,CAACC,OAAO,CAACC,SAAS,CAAC,CAAC,EAAE;QAC7C,OAAO,IAAI,CAACjB,OAAO,CAACe,UAAU,CAACG,QAAQ,CAACC,OAAO,CAAC,CAAC;MACrD;MACA;AACZ;AACA;AACA;MACY;MAAA,KACK,IAAI,IAAI,CAACnB,OAAO,CAACe,UAAU,CAACC,OAAO,CAACI,gBAAgB,CAAC,GAAG,CAAC,EAAE;QAC5D,OAAO,IAAI,CAACpB,OAAO,CAACe,UAAU,CAACG,QAAQ,CAACG,QAAQ,CAAC;UAC7C,GAAGhB,KAAK;UACRE,uBAAuB,EAAE;YACrB,GAAGF,KAAK,CAACE,uBAAuB;YAChCC,KAAK,EAAEM,YAAY;YACnBL;UACJ;QACJ,CAAC,CAAC;MACN;MAEA,MAAMa,MAAM,GAAG,MAAM,IAAI,CAACnB,OAAO,CAACoB,KAAK,CAAC;QACpCf,KAAK,EAAEM,YAAY;QACnBL,MAAM;QACNe,KAAK,EAAE;MACX,CAAC,CAAC;MAEF,MAAMC,UAAU,GAAG,MAAM,IAAI,CAACvB,WAAW,CAACwB,OAAO,CAAC;QAC9CC,IAAI,EAAEL,MAAM,CAACK,IAAI;QACjBnB,KAAK,EAAEM,YAAY;QACnBc,KAAK,EAAEN,MAAM,CAACM;MAClB,CAAC,CAAC;MAEF,IAAI,CAACH,UAAU,CAACE,IAAI,IAAIL,MAAM,CAACb,MAAM,EAAE;QACnCA,MAAM,GAAGa,MAAM,CAACb,MAAM;QACtB;MACJ;MACAA,MAAM,GAAGoB,SAAS;MAElB,MAAMjB,IAAI,GAAGF,OAAO,CAACG,SAAS,CAACL,KAAK,IAAIA,KAAK,KAAKM,YAAY,CAAC,GAAG,CAAC;MACnEA,YAAY,GAAGJ,OAAO,CAACE,IAAI,CAAC;IAChC;IAEA,OAAO,IAAI,CAACZ,OAAO,CAACe,UAAU,CAACG,QAAQ,CAACG,QAAQ,CAAC;MAC7C,GAAGhB,KAAK;MACRE,uBAAuB,EAAE;QACrBuB,QAAQ,EAAE;MACd;IACJ,CAAC,CAAC;EACN;EAEA,MAAcnB,eAAeA,CAAA,EAAmC;IAC5D,MAAMW,MAAM,GAAG,MAAM,IAAI,CAACrB,YAAY,CAAC8B,IAAI,CAAC,CAAC;IAC7C,IAAIT,MAAM,CAACU,MAAM,GAAG,CAAC,EAAE;MACnB,OAAOV,MAAM;IACjB;IACA,MAAM,IAAIW,KAAK,CAAC,8CAA8C,CAAC;EACnE;AACJ","ignoreList":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { PrimitiveValue } from \"@webiny/api-elasticsearch/types\";\n\nexport interface IElasticsearchFetcherFetchResponseItem {\n PK: string;\n SK: string;\n _id: string;\n index: string;\n}\n\nexport interface IElasticsearchFetcherFetchParams {\n index: string;\n cursor?: PrimitiveValue[];\n limit: number;\n}\n\nexport interface IElasticsearchFetcherFetchResponse {\n done: boolean;\n totalCount: number;\n cursor?: PrimitiveValue[];\n items: IElasticsearchFetcherFetchResponseItem[];\n}\n\nexport interface IElasticsearchFetcher {\n fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;\n}\n"],"mappings":"","ignoreList":[]}
|
|
1
|
+
{"version":3,"names":[],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { PrimitiveValue } from \"@webiny/api-elasticsearch/types.js\";\n\nexport interface IElasticsearchFetcherFetchResponseItem {\n PK: string;\n SK: string;\n _id: string;\n index: string;\n}\n\nexport interface IElasticsearchFetcherFetchParams {\n index: string;\n cursor?: PrimitiveValue[];\n limit: number;\n}\n\nexport interface IElasticsearchFetcherFetchResponse {\n done: boolean;\n totalCount: number;\n cursor?: PrimitiveValue[];\n items: IElasticsearchFetcherFetchResponseItem[];\n}\n\nexport interface IElasticsearchFetcher {\n fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;\n}\n"],"mappings":"","ignoreList":[]}
|
|
@@ -1,13 +1,6 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.shouldIgnoreEsResponseError = void 0;
|
|
7
1
|
const IGNORED_ES_SEARCH_EXCEPTIONS = ["index_not_found_exception", "search_phase_execution_exception", "illegal_argument_exception"];
|
|
8
|
-
const shouldIgnoreEsResponseError = error => {
|
|
2
|
+
export const shouldIgnoreEsResponseError = error => {
|
|
9
3
|
return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message);
|
|
10
4
|
};
|
|
11
|
-
exports.shouldIgnoreEsResponseError = shouldIgnoreEsResponseError;
|
|
12
5
|
|
|
13
6
|
//# sourceMappingURL=shouldIgnoreEsResponseError.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["IGNORED_ES_SEARCH_EXCEPTIONS","shouldIgnoreEsResponseError","error","includes","message"
|
|
1
|
+
{"version":3,"names":["IGNORED_ES_SEARCH_EXCEPTIONS","shouldIgnoreEsResponseError","error","includes","message"],"sources":["shouldIgnoreEsResponseError.ts"],"sourcesContent":["import type WebinyError from \"@webiny/error\";\n\nconst IGNORED_ES_SEARCH_EXCEPTIONS = [\n \"index_not_found_exception\",\n \"search_phase_execution_exception\",\n \"illegal_argument_exception\"\n];\n\nexport const shouldIgnoreEsResponseError = (error: WebinyError) => {\n return IGNORED_ES_SEARCH_EXCEPTIONS.includes(error.message);\n};\n"],"mappings":"AAEA,MAAMA,4BAA4B,GAAG,CACjC,2BAA2B,EAC3B,kCAAkC,EAClC,4BAA4B,CAC/B;AAED,OAAO,MAAMC,2BAA2B,GAAIC,KAAkB,IAAK;EAC/D,OAAOF,4BAA4B,CAACG,QAAQ,CAACD,KAAK,CAACE,OAAO,CAAC;AAC/D,CAAC","ignoreList":[]}
|