@webiny/api-elasticsearch-tasks 6.0.0-beta.0 → 6.0.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/abstractions/DbRegistry.d.ts +11 -0
- package/abstractions/DbRegistry.js +4 -0
- package/abstractions/DbRegistry.js.map +1 -0
- package/abstractions/OpensearchTenantIndexFactory.d.ts +14 -0
- package/abstractions/OpensearchTenantIndexFactory.js +4 -0
- package/abstractions/OpensearchTenantIndexFactory.js.map +1 -0
- package/abstractions/SynchronizationContext.d.ts +6 -0
- package/abstractions/SynchronizationContext.js +4 -0
- package/abstractions/SynchronizationContext.js.map +1 -0
- package/errors/IndexSettingsGetError.d.ts +1 -1
- package/errors/IndexSettingsGetError.js +2 -10
- package/errors/IndexSettingsGetError.js.map +1 -1
- package/errors/IndexSettingsSetError.d.ts +1 -1
- package/errors/IndexSettingsSetError.js +2 -10
- package/errors/IndexSettingsSetError.js.map +1 -1
- package/errors/IndexingDisableError.js +1 -8
- package/errors/IndexingDisableError.js.map +1 -1
- package/errors/IndexingEnableError.js +1 -8
- package/errors/IndexingEnableError.js.map +1 -1
- package/errors/index.d.ts +4 -4
- package/errors/index.js +4 -49
- package/errors/index.js.map +1 -1
- package/helpers/getClients.d.ts +5 -0
- package/helpers/getClients.js +14 -0
- package/helpers/getClients.js.map +1 -0
- package/helpers/scan.d.ts +5 -5
- package/helpers/scan.js +5 -12
- package/helpers/scan.js.map +1 -1
- package/index.d.ts +5 -5
- package/index.js +14 -25
- package/index.js.map +1 -1
- package/package.json +33 -34
- package/settings/DisableIndexing.d.ts +2 -2
- package/settings/DisableIndexing.js +3 -10
- package/settings/DisableIndexing.js.map +1 -1
- package/settings/EnableIndexing.d.ts +2 -2
- package/settings/EnableIndexing.js +5 -11
- package/settings/EnableIndexing.js.map +1 -1
- package/settings/IndexManager.d.ts +6 -3
- package/settings/IndexManager.js +21 -17
- package/settings/IndexManager.js.map +1 -1
- package/settings/IndexSettingsManager.d.ts +2 -2
- package/settings/IndexSettingsManager.js +4 -11
- package/settings/IndexSettingsManager.js.map +1 -1
- package/settings/index.d.ts +1 -1
- package/settings/index.js +1 -16
- package/settings/index.js.map +1 -1
- package/settings/types.d.ts +1 -1
- package/settings/types.js +1 -5
- package/settings/types.js.map +1 -1
- package/tasks/Manager.d.ts +17 -26
- package/tasks/Manager.js +10 -36
- package/tasks/Manager.js.map +1 -1
- package/tasks/createIndexes/CreateIndexesTask.d.ts +23 -0
- package/tasks/createIndexes/CreateIndexesTask.js +53 -0
- package/tasks/createIndexes/CreateIndexesTask.js.map +1 -0
- package/tasks/createIndexes/CreateIndexesTaskRunner.d.ts +14 -7
- package/tasks/createIndexes/CreateIndexesTaskRunner.js +23 -38
- package/tasks/createIndexes/CreateIndexesTaskRunner.js.map +1 -1
- package/tasks/createIndexes/OnBeforeTrigger.d.ts +10 -0
- package/tasks/createIndexes/OnBeforeTrigger.js +51 -0
- package/tasks/createIndexes/OnBeforeTrigger.js.map +1 -0
- package/tasks/createIndexes/createIndex.d.ts +5 -0
- package/tasks/createIndexes/createIndex.js +20 -0
- package/tasks/createIndexes/createIndex.js.map +1 -0
- package/tasks/createIndexes/index.d.ts +2 -3
- package/tasks/createIndexes/index.js +17 -44
- package/tasks/createIndexes/index.js.map +1 -1
- package/tasks/createIndexes/listIndexes.d.ts +4 -0
- package/tasks/createIndexes/listIndexes.js +20 -0
- package/tasks/createIndexes/listIndexes.js.map +1 -0
- package/tasks/createIndexes/types.js +1 -5
- package/tasks/dataSynchronization/DataSynchronizationTask.d.ts +34 -0
- package/tasks/dataSynchronization/DataSynchronizationTask.js +69 -0
- package/tasks/dataSynchronization/DataSynchronizationTask.js.map +1 -0
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.d.ts +18 -0
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js +45 -0
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js.map +1 -0
- package/tasks/dataSynchronization/createFactories.d.ts +2 -0
- package/tasks/dataSynchronization/createFactories.js +10 -0
- package/tasks/dataSynchronization/createFactories.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.d.ts +10 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js +91 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.d.ts +12 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js +70 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.d.ts +10 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js +70 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.d.ts +21 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js +3 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.d.ts +17 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js +3 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.d.ts +2 -0
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js +6 -0
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js.map +1 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.d.ts +10 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.js +24 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.js.map +1 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.d.ts +8 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js +12 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js.map +1 -0
- package/tasks/dataSynchronization/entities/getTable.d.ts +7 -0
- package/tasks/dataSynchronization/entities/getTable.js +22 -0
- package/tasks/dataSynchronization/entities/getTable.js.map +1 -0
- package/tasks/dataSynchronization/entities/index.d.ts +3 -0
- package/tasks/dataSynchronization/entities/index.js +5 -0
- package/tasks/dataSynchronization/entities/index.js.map +1 -0
- package/tasks/dataSynchronization/index.d.ts +3 -0
- package/tasks/dataSynchronization/index.js +28 -0
- package/tasks/dataSynchronization/index.js.map +1 -0
- package/tasks/dataSynchronization/types.d.ts +39 -0
- package/tasks/dataSynchronization/types.js +3 -0
- package/tasks/dataSynchronization/types.js.map +1 -0
- package/tasks/enableIndexing/EnableIndexingTaskRunner.d.ts +6 -6
- package/tasks/enableIndexing/EnableIndexingTaskRunner.js +5 -13
- package/tasks/enableIndexing/EnableIndexingTaskRunner.js.map +1 -1
- package/tasks/enableIndexing/index.d.ts +2 -3
- package/tasks/enableIndexing/index.js +40 -44
- package/tasks/enableIndexing/index.js.map +1 -1
- package/tasks/enableIndexing/types.js +1 -5
- package/tasks/index.d.ts +4 -3
- package/tasks/index.js +4 -38
- package/tasks/index.js.map +1 -1
- package/tasks/reindexing/ReindexingTaskRunner.d.ts +5 -6
- package/tasks/reindexing/ReindexingTaskRunner.js +23 -27
- package/tasks/reindexing/ReindexingTaskRunner.js.map +1 -1
- package/tasks/reindexing/index.d.ts +1 -1
- package/tasks/reindexing/index.js +1 -16
- package/tasks/reindexing/index.js.map +1 -1
- package/tasks/reindexing/reindexingTaskDefinition.d.ts +2 -2
- package/tasks/reindexing/reindexingTaskDefinition.js +43 -42
- package/tasks/reindexing/reindexingTaskDefinition.js.map +1 -1
- package/types.d.ts +22 -22
- package/types.js +1 -5
- package/types.js.map +1 -1
- package/definitions/entry.d.ts +0 -7
- package/definitions/entry.js +0 -36
- package/definitions/entry.js.map +0 -1
- package/definitions/index.d.ts +0 -2
- package/definitions/index.js +0 -29
- package/definitions/index.js.map +0 -1
- package/definitions/table.d.ts +0 -7
- package/definitions/table.js +0 -23
- package/definitions/table.js.map +0 -1
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.d.ts +0 -23
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.js +0 -25
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.js.map +0 -1
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { IManager } from "../../types";
|
|
2
|
-
import {
|
|
3
|
-
import { IndexManager } from "../../settings";
|
|
1
|
+
import type { IManager } from "../../types.js";
|
|
2
|
+
import type { ITaskResult } from "@webiny/api-core/features/task/TaskDefinition/index.js";
|
|
3
|
+
import type { IndexManager } from "../../settings/index.js";
|
|
4
|
+
import type { IElasticsearchEnableIndexingTaskInput } from "../../tasks/enableIndexing/types.js";
|
|
4
5
|
export declare class EnableIndexingTaskRunner {
|
|
5
6
|
private readonly manager;
|
|
6
7
|
private readonly indexManager;
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
exec(matching?: string): Promise<ITaskResponseResult>;
|
|
8
|
+
constructor(manager: IManager<IElasticsearchEnableIndexingTaskInput>, indexManager: IndexManager);
|
|
9
|
+
exec(matching?: string): Promise<ITaskResult<IElasticsearchEnableIndexingTaskInput>>;
|
|
10
10
|
}
|
|
@@ -1,18 +1,11 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.EnableIndexingTaskRunner = void 0;
|
|
7
|
-
class EnableIndexingTaskRunner {
|
|
1
|
+
export class EnableIndexingTaskRunner {
|
|
8
2
|
constructor(manager, indexManager) {
|
|
9
3
|
this.manager = manager;
|
|
10
|
-
this.response = manager.response;
|
|
11
4
|
this.indexManager = indexManager;
|
|
12
5
|
}
|
|
13
6
|
async exec(matching) {
|
|
14
|
-
if (this.manager.isAborted()) {
|
|
15
|
-
return this.response.aborted();
|
|
7
|
+
if (this.manager.controller.runtime.isAborted()) {
|
|
8
|
+
return this.manager.controller.response.aborted();
|
|
16
9
|
}
|
|
17
10
|
const isIndexAllowed = index => {
|
|
18
11
|
if (typeof matching !== "string" || !matching) {
|
|
@@ -32,18 +25,17 @@ class EnableIndexingTaskRunner {
|
|
|
32
25
|
enabled.push(index);
|
|
33
26
|
} catch (ex) {
|
|
34
27
|
failed.push(index);
|
|
35
|
-
await this.manager.
|
|
28
|
+
await this.manager.controller.logger.error({
|
|
36
29
|
message: `Failed to enable indexing on index "${index}".`,
|
|
37
30
|
error: ex
|
|
38
31
|
});
|
|
39
32
|
}
|
|
40
33
|
}
|
|
41
|
-
return this.response.done("Task done.", {
|
|
34
|
+
return this.manager.controller.response.done("Task done.", {
|
|
42
35
|
enabled,
|
|
43
36
|
failed
|
|
44
37
|
});
|
|
45
38
|
}
|
|
46
39
|
}
|
|
47
|
-
exports.EnableIndexingTaskRunner = EnableIndexingTaskRunner;
|
|
48
40
|
|
|
49
41
|
//# sourceMappingURL=EnableIndexingTaskRunner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["EnableIndexingTaskRunner","constructor","manager","indexManager","
|
|
1
|
+
{"version":3,"names":["EnableIndexingTaskRunner","constructor","manager","indexManager","exec","matching","controller","runtime","isAborted","response","aborted","isIndexAllowed","index","includes","indexes","list","enabled","failed","enableIndexing","push","ex","logger","error","message","done"],"sources":["EnableIndexingTaskRunner.ts"],"sourcesContent":["import type { IManager } from \"~/types.js\";\nimport type { ITaskResult } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport type { IndexManager } from \"~/settings/index.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { IElasticsearchEnableIndexingTaskInput } from \"~/tasks/enableIndexing/types.js\";\n\nexport class EnableIndexingTaskRunner {\n private readonly manager: IManager<IElasticsearchEnableIndexingTaskInput>;\n private readonly indexManager: IIndexManager;\n\n public constructor(\n manager: IManager<IElasticsearchEnableIndexingTaskInput>,\n indexManager: IndexManager\n ) {\n this.manager = manager;\n this.indexManager = indexManager;\n }\n\n public async exec(\n matching?: string\n ): Promise<ITaskResult<IElasticsearchEnableIndexingTaskInput>> {\n if (this.manager.controller.runtime.isAborted()) {\n return this.manager.controller.response.aborted();\n }\n\n const isIndexAllowed = (index: string): boolean => {\n if (typeof matching !== \"string\" || !matching) {\n return true;\n }\n return index.includes(matching);\n };\n\n const indexes = await this.indexManager.list();\n const enabled: string[] = [];\n const failed: string[] = [];\n for (const index of indexes) {\n if (!isIndexAllowed(index)) {\n continue;\n }\n try {\n await this.indexManager.enableIndexing(index);\n enabled.push(index);\n } catch (ex) {\n failed.push(index);\n await this.manager.controller.logger.error({\n message: `Failed to enable indexing on index \"${index}\".`,\n error: ex\n });\n }\n }\n return this.manager.controller.response.done(\"Task done.\", {\n enabled,\n failed\n });\n }\n}\n"],"mappings":"AAMA,OAAO,MAAMA,wBAAwB,CAAC;EAI3BC,WAAWA,CACdC,OAAwD,EACxDC,YAA0B,EAC5B;IACE,IAAI,CAACD,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,YAAY,GAAGA,YAAY;EACpC;EAEA,MAAaC,IAAIA,CACbC,QAAiB,EAC0C;IAC3D,IAAI,IAAI,CAACH,OAAO,CAACI,UAAU,CAACC,OAAO,CAACC,SAAS,CAAC,CAAC,EAAE;MAC7C,OAAO,IAAI,CAACN,OAAO,CAACI,UAAU,CAACG,QAAQ,CAACC,OAAO,CAAC,CAAC;IACrD;IAEA,MAAMC,cAAc,GAAIC,KAAa,IAAc;MAC/C,IAAI,OAAOP,QAAQ,KAAK,QAAQ,IAAI,CAACA,QAAQ,EAAE;QAC3C,OAAO,IAAI;MACf;MACA,OAAOO,KAAK,CAACC,QAAQ,CAACR,QAAQ,CAAC;IACnC,CAAC;IAED,MAAMS,OAAO,GAAG,MAAM,IAAI,CAACX,YAAY,CAACY,IAAI,CAAC,CAAC;IAC9C,MAAMC,OAAiB,GAAG,EAAE;IAC5B,MAAMC,MAAgB,GAAG,EAAE;IAC3B,KAAK,MAAML,KAAK,IAAIE,OAAO,EAAE;MACzB,IAAI,CAACH,cAAc,CAACC,KAAK,CAAC,EAAE;QACxB;MACJ;MACA,IAAI;QACA,MAAM,IAAI,CAACT,YAAY,CAACe,cAAc,CAACN,KAAK,CAAC;QAC7CI,OAAO,CAACG,IAAI,CAACP,KAAK,CAAC;MACvB,CAAC,CAAC,OAAOQ,EAAE,EAAE;QACTH,MAAM,CAACE,IAAI,CAACP,KAAK,CAAC;QAClB,MAAM,IAAI,CAACV,OAAO,CAACI,UAAU,CAACe,MAAM,CAACC,KAAK,CAAC;UACvCC,OAAO,EAAE,uCAAuCX,KAAK,IAAI;UACzDU,KAAK,EAAEF;QACX,CAAC,CAAC;MACN;IACJ;IACA,OAAO,IAAI,CAAClB,OAAO,CAACI,UAAU,CAACG,QAAQ,CAACe,IAAI,CAAC,YAAY,EAAE;MACvDR,OAAO;MACPC;IACJ,CAAC,CAAC;EACN;AACJ","ignoreList":[]}
|
|
@@ -1,3 +1,2 @@
|
|
|
1
|
-
import { Context, IElasticsearchTaskConfig } from "../../types";
|
|
2
|
-
|
|
3
|
-
export declare const createEnableIndexingTask: (params?: IElasticsearchTaskConfig) => import("@webiny/tasks").TaskDefinitionPlugin<Context, IElasticsearchEnableIndexingTaskInput, import("@webiny/tasks").ITaskResponseDoneResultOutput>;
|
|
1
|
+
import type { Context, IElasticsearchTaskConfig } from "../../types.js";
|
|
2
|
+
export declare const createEnableIndexingTask: (params?: Partial<IElasticsearchTaskConfig>) => import("@webiny/api").ContextPlugin<Context>;
|
|
@@ -1,49 +1,45 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
}
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
const {
|
|
21
|
-
Manager
|
|
22
|
-
} = await import( /* webpackChunkName: "ElasticsearchTaskManager" */"../Manager");
|
|
23
|
-
const {
|
|
24
|
-
IndexManager
|
|
25
|
-
} = await import( /* webpackChunkName: "ElasticsearchTaskSettings" */"../../settings");
|
|
26
|
-
const {
|
|
27
|
-
EnableIndexingTaskRunner
|
|
28
|
-
} = await import( /* webpackChunkName: "ElasticsearchEnableIndexingTaskRunner" */"./EnableIndexingTaskRunner");
|
|
29
|
-
const manager = new Manager({
|
|
30
|
-
elasticsearchClient: params?.elasticsearchClient,
|
|
31
|
-
documentClient: params?.documentClient,
|
|
32
|
-
response,
|
|
33
|
-
context,
|
|
34
|
-
isAborted,
|
|
35
|
-
isCloseToTimeout,
|
|
36
|
-
store
|
|
37
|
-
});
|
|
38
|
-
const indexManager = new IndexManager(manager.elasticsearch, {}, {
|
|
39
|
-
refreshInterval: input.refreshInterval,
|
|
40
|
-
numberOfReplicas: input.numberOfReplicas
|
|
41
|
-
});
|
|
42
|
-
const enableIndexing = new EnableIndexingTaskRunner(manager, indexManager);
|
|
43
|
-
return enableIndexing.exec(input.matching);
|
|
1
|
+
import { createContextPlugin } from "@webiny/api";
|
|
2
|
+
import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js";
|
|
3
|
+
import { Manager } from "../Manager.js";
|
|
4
|
+
import { IndexManager } from "../../settings/index.js";
|
|
5
|
+
import { EnableIndexingTaskRunner } from "./EnableIndexingTaskRunner.js";
|
|
6
|
+
import { getClients } from "../../helpers/getClients.js";
|
|
7
|
+
class ElasticsearchEnableIndexingTask {
|
|
8
|
+
id = "elasticsearchEnableIndexing";
|
|
9
|
+
title = "Enable Indexing on Elasticsearch Indexes";
|
|
10
|
+
constructor(elasticsearchClient, documentClient) {
|
|
11
|
+
this.elasticsearchClient = elasticsearchClient;
|
|
12
|
+
this.documentClient = documentClient;
|
|
13
|
+
}
|
|
14
|
+
async run({
|
|
15
|
+
input,
|
|
16
|
+
controller
|
|
17
|
+
}) {
|
|
18
|
+
if (controller.runtime.isAborted()) {
|
|
19
|
+
return controller.response.aborted();
|
|
44
20
|
}
|
|
21
|
+
const manager = new Manager({
|
|
22
|
+
elasticsearchClient: this.elasticsearchClient,
|
|
23
|
+
documentClient: this.documentClient,
|
|
24
|
+
controller
|
|
25
|
+
});
|
|
26
|
+
const indexManager = new IndexManager(manager.elasticsearch, {}, {
|
|
27
|
+
refreshInterval: input.refreshInterval,
|
|
28
|
+
numberOfReplicas: input.numberOfReplicas
|
|
29
|
+
});
|
|
30
|
+
const enableIndexing = new EnableIndexingTaskRunner(manager, indexManager);
|
|
31
|
+
return enableIndexing.exec(input.matching);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
export const createEnableIndexingTask = params => {
|
|
35
|
+
return createContextPlugin(context => {
|
|
36
|
+
const clients = getClients(context, params);
|
|
37
|
+
|
|
38
|
+
// Register the task definition
|
|
39
|
+
context.container.registerFactory(TaskDefinition, () => {
|
|
40
|
+
return new ElasticsearchEnableIndexingTask(clients.elasticsearchClient, clients.documentClient);
|
|
41
|
+
});
|
|
45
42
|
});
|
|
46
43
|
};
|
|
47
|
-
exports.createEnableIndexingTask = createEnableIndexingTask;
|
|
48
44
|
|
|
49
45
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["createContextPlugin","TaskDefinition","Manager","IndexManager","EnableIndexingTaskRunner","getClients","ElasticsearchEnableIndexingTask","id","title","constructor","elasticsearchClient","documentClient","run","input","controller","runtime","isAborted","response","aborted","manager","indexManager","elasticsearch","refreshInterval","numberOfReplicas","enableIndexing","exec","matching","createEnableIndexingTask","params","context","clients","container","registerFactory"],"sources":["index.ts"],"sourcesContent":["import { createContextPlugin } from \"@webiny/api\";\nimport { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport type { Context, IElasticsearchTaskConfig } from \"~/types.js\";\nimport type { IElasticsearchEnableIndexingTaskInput } from \"./types.js\";\nimport { Manager } from \"../Manager.js\";\nimport { IndexManager } from \"~/settings/index.js\";\nimport { EnableIndexingTaskRunner } from \"./EnableIndexingTaskRunner.js\";\nimport { getClients } from \"~/helpers/getClients.js\";\n\nclass ElasticsearchEnableIndexingTask\n implements TaskDefinition.Interface<IElasticsearchEnableIndexingTaskInput>\n{\n id = \"elasticsearchEnableIndexing\";\n title = \"Enable Indexing on Elasticsearch Indexes\";\n\n constructor(\n private elasticsearchClient: IElasticsearchTaskConfig[\"elasticsearchClient\"],\n private documentClient: IElasticsearchTaskConfig[\"documentClient\"]\n ) {}\n\n async run({\n input,\n controller\n }: TaskDefinition.RunParams<IElasticsearchEnableIndexingTaskInput>) {\n if (controller.runtime.isAborted()) {\n return controller.response.aborted();\n }\n\n const manager = new Manager<IElasticsearchEnableIndexingTaskInput>({\n elasticsearchClient: this.elasticsearchClient,\n documentClient: this.documentClient,\n controller\n });\n\n const indexManager = new IndexManager(\n manager.elasticsearch,\n {},\n {\n refreshInterval: input.refreshInterval,\n numberOfReplicas: input.numberOfReplicas\n }\n );\n\n const enableIndexing = new EnableIndexingTaskRunner(manager, indexManager);\n\n return enableIndexing.exec(input.matching);\n }\n}\n\nexport const createEnableIndexingTask = (params?: Partial<IElasticsearchTaskConfig>) => {\n return createContextPlugin<Context>(context => {\n const clients = getClients(context, params);\n\n // Register the task definition\n context.container.registerFactory(TaskDefinition, () => {\n return new ElasticsearchEnableIndexingTask(\n clients.elasticsearchClient,\n clients.documentClient\n );\n });\n });\n};\n"],"mappings":"AAAA,SAASA,mBAAmB,QAAQ,aAAa;AACjD,SAASC,cAAc,QAAQ,wDAAwD;AAGvF,SAASC,OAAO;AAChB,SAASC,YAAY;AACrB,SAASC,wBAAwB;AACjC,SAASC,UAAU;AAEnB,MAAMC,+BAA+B,CAErC;EACIC,EAAE,GAAG,6BAA6B;EAClCC,KAAK,GAAG,0CAA0C;EAElDC,WAAWA,CACCC,mBAAoE,EACpEC,cAA0D,EACpE;IAAA,KAFUD,mBAAoE,GAApEA,mBAAoE;IAAA,KACpEC,cAA0D,GAA1DA,cAA0D;EACnE;EAEH,MAAMC,GAAGA,CAAC;IACNC,KAAK;IACLC;EAC6D,CAAC,EAAE;IAChE,IAAIA,UAAU,CAACC,OAAO,CAACC,SAAS,CAAC,CAAC,EAAE;MAChC,OAAOF,UAAU,CAACG,QAAQ,CAACC,OAAO,CAAC,CAAC;IACxC;IAEA,MAAMC,OAAO,GAAG,IAAIjB,OAAO,CAAwC;MAC/DQ,mBAAmB,EAAE,IAAI,CAACA,mBAAmB;MAC7CC,cAAc,EAAE,IAAI,CAACA,cAAc;MACnCG;IACJ,CAAC,CAAC;IAEF,MAAMM,YAAY,GAAG,IAAIjB,YAAY,CACjCgB,OAAO,CAACE,aAAa,EACrB,CAAC,CAAC,EACF;MACIC,eAAe,EAAET,KAAK,CAACS,eAAe;MACtCC,gBAAgB,EAAEV,KAAK,CAACU;IAC5B,CACJ,CAAC;IAED,MAAMC,cAAc,GAAG,IAAIpB,wBAAwB,CAACe,OAAO,EAAEC,YAAY,CAAC;IAE1E,OAAOI,cAAc,CAACC,IAAI,CAACZ,KAAK,CAACa,QAAQ,CAAC;EAC9C;AACJ;AAEA,OAAO,MAAMC,wBAAwB,GAAIC,MAA0C,IAAK;EACpF,OAAO5B,mBAAmB,CAAU6B,OAAO,IAAI;IAC3C,MAAMC,OAAO,GAAGzB,UAAU,CAACwB,OAAO,EAAED,MAAM,CAAC;;IAE3C;IACAC,OAAO,CAACE,SAAS,CAACC,eAAe,CAAC/B,cAAc,EAAE,MAAM;MACpD,OAAO,IAAIK,+BAA+B,CACtCwB,OAAO,CAACpB,mBAAmB,EAC3BoB,OAAO,CAACnB,cACZ,CAAC;IACL,CAAC,CAAC;EACN,CAAC,CAAC;AACN,CAAC","ignoreList":[]}
|
package/tasks/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
-
export * from "./enableIndexing";
|
|
2
|
-
export * from "./
|
|
3
|
-
export * from "./
|
|
1
|
+
export * from "./enableIndexing/index.js";
|
|
2
|
+
export * from "./dataSynchronization/index.js";
|
|
3
|
+
export * from "./reindexing/index.js";
|
|
4
|
+
export * from "./createIndexes/index.js";
|
package/tasks/index.js
CHANGED
|
@@ -1,40 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
});
|
|
6
|
-
var _enableIndexing = require("./enableIndexing");
|
|
7
|
-
Object.keys(_enableIndexing).forEach(function (key) {
|
|
8
|
-
if (key === "default" || key === "__esModule") return;
|
|
9
|
-
if (key in exports && exports[key] === _enableIndexing[key]) return;
|
|
10
|
-
Object.defineProperty(exports, key, {
|
|
11
|
-
enumerable: true,
|
|
12
|
-
get: function () {
|
|
13
|
-
return _enableIndexing[key];
|
|
14
|
-
}
|
|
15
|
-
});
|
|
16
|
-
});
|
|
17
|
-
var _reindexing = require("./reindexing");
|
|
18
|
-
Object.keys(_reindexing).forEach(function (key) {
|
|
19
|
-
if (key === "default" || key === "__esModule") return;
|
|
20
|
-
if (key in exports && exports[key] === _reindexing[key]) return;
|
|
21
|
-
Object.defineProperty(exports, key, {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
get: function () {
|
|
24
|
-
return _reindexing[key];
|
|
25
|
-
}
|
|
26
|
-
});
|
|
27
|
-
});
|
|
28
|
-
var _createIndexes = require("./createIndexes");
|
|
29
|
-
Object.keys(_createIndexes).forEach(function (key) {
|
|
30
|
-
if (key === "default" || key === "__esModule") return;
|
|
31
|
-
if (key in exports && exports[key] === _createIndexes[key]) return;
|
|
32
|
-
Object.defineProperty(exports, key, {
|
|
33
|
-
enumerable: true,
|
|
34
|
-
get: function () {
|
|
35
|
-
return _createIndexes[key];
|
|
36
|
-
}
|
|
37
|
-
});
|
|
38
|
-
});
|
|
1
|
+
export * from "./enableIndexing/index.js";
|
|
2
|
+
export * from "./dataSynchronization/index.js";
|
|
3
|
+
export * from "./reindexing/index.js";
|
|
4
|
+
export * from "./createIndexes/index.js";
|
|
39
5
|
|
|
40
6
|
//# sourceMappingURL=index.js.map
|
package/tasks/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[
|
|
1
|
+
{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./enableIndexing/index.js\";\nexport * from \"./dataSynchronization/index.js\";\nexport * from \"./reindexing/index.js\";\nexport * from \"./createIndexes/index.js\";\n"],"mappings":"AAAA;AACA;AACA;AACA","ignoreList":[]}
|
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
import { IElasticsearchIndexingTaskValuesKeys, IManager } from "../../types";
|
|
2
|
-
import {
|
|
3
|
-
import { IndexManager } from "../../settings";
|
|
1
|
+
import type { IElasticsearchIndexingTaskValues, IElasticsearchIndexingTaskValuesKeys, IManager } from "../../types.js";
|
|
2
|
+
import type { ITaskResult } from "@webiny/api-core/features/task/TaskDefinition/index.js";
|
|
3
|
+
import type { IndexManager } from "../../settings/index.js";
|
|
4
4
|
export declare class ReindexingTaskRunner {
|
|
5
5
|
private readonly manager;
|
|
6
6
|
private keys?;
|
|
7
7
|
private readonly indexManager;
|
|
8
|
-
|
|
9
|
-
constructor(manager: IManager, indexManager: IndexManager);
|
|
8
|
+
constructor(manager: IManager<IElasticsearchIndexingTaskValues>, indexManager: IndexManager);
|
|
10
9
|
/**
|
|
11
10
|
* When running the task, we always must check:
|
|
12
11
|
* * if task is close to timeout
|
|
13
12
|
* * if task was aborted
|
|
14
13
|
*/
|
|
15
|
-
exec(keys: IElasticsearchIndexingTaskValuesKeys | undefined, limit: number): Promise<
|
|
14
|
+
exec(keys: IElasticsearchIndexingTaskValuesKeys | undefined, limit: number): Promise<ITaskResult<IElasticsearchIndexingTaskValues>>;
|
|
16
15
|
}
|
|
@@ -1,10 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
exports.ReindexingTaskRunner = void 0;
|
|
7
|
-
var _scan = require("../../helpers/scan");
|
|
1
|
+
import { scan } from "../../helpers/scan.js";
|
|
2
|
+
import { createTableWriteBatch } from "@webiny/db-dynamodb";
|
|
8
3
|
const getKeys = results => {
|
|
9
4
|
if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {
|
|
10
5
|
return {
|
|
@@ -14,10 +9,9 @@ const getKeys = results => {
|
|
|
14
9
|
}
|
|
15
10
|
return undefined;
|
|
16
11
|
};
|
|
17
|
-
class ReindexingTaskRunner {
|
|
12
|
+
export class ReindexingTaskRunner {
|
|
18
13
|
constructor(manager, indexManager) {
|
|
19
14
|
this.manager = manager;
|
|
20
|
-
this.response = manager.response;
|
|
21
15
|
this.indexManager = indexManager;
|
|
22
16
|
}
|
|
23
17
|
|
|
@@ -29,19 +23,19 @@ class ReindexingTaskRunner {
|
|
|
29
23
|
async exec(keys = undefined, limit) {
|
|
30
24
|
this.keys = keys;
|
|
31
25
|
const isIndexAllowed = index => {
|
|
32
|
-
const input = this.manager.
|
|
26
|
+
const input = this.manager.controller.state.getInput();
|
|
33
27
|
if (typeof input.matching !== "string" || !input.matching) {
|
|
34
28
|
return true;
|
|
35
29
|
}
|
|
36
30
|
return index.includes(input.matching);
|
|
37
31
|
};
|
|
38
32
|
try {
|
|
39
|
-
while (this.manager.isCloseToTimeout() === false) {
|
|
40
|
-
if (this.manager.isAborted()) {
|
|
41
|
-
return this.response.aborted();
|
|
33
|
+
while (this.manager.controller.runtime.isCloseToTimeout() === false) {
|
|
34
|
+
if (this.manager.controller.runtime.isAborted()) {
|
|
35
|
+
return this.manager.controller.response.aborted();
|
|
42
36
|
}
|
|
43
|
-
const results = await
|
|
44
|
-
table: this.manager.table,
|
|
37
|
+
const results = await scan({
|
|
38
|
+
table: this.manager.table.table,
|
|
45
39
|
keys: this.keys,
|
|
46
40
|
options: {
|
|
47
41
|
limit
|
|
@@ -49,9 +43,11 @@ class ReindexingTaskRunner {
|
|
|
49
43
|
});
|
|
50
44
|
if (results.items.length === 0) {
|
|
51
45
|
await this.indexManager.enableIndexing();
|
|
52
|
-
return this.response.done("No more items to process.");
|
|
46
|
+
return this.manager.controller.response.done("No more items to process.");
|
|
53
47
|
}
|
|
54
|
-
const
|
|
48
|
+
const tableWriteBatch = createTableWriteBatch({
|
|
49
|
+
table: this.manager.table.table
|
|
50
|
+
});
|
|
55
51
|
for (const item of results.items) {
|
|
56
52
|
/**
|
|
57
53
|
* No index defined? Impossible but let's skip if really happens.
|
|
@@ -64,7 +60,7 @@ class ReindexingTaskRunner {
|
|
|
64
60
|
}
|
|
65
61
|
const exists = await this.indexManager.indexExists(item.index);
|
|
66
62
|
if (!exists) {
|
|
67
|
-
await this.manager.
|
|
63
|
+
await this.manager.controller.logger.info({
|
|
68
64
|
message: `Index "${item.index}" does not exist. Skipping the item.`
|
|
69
65
|
});
|
|
70
66
|
continue;
|
|
@@ -88,18 +84,19 @@ class ReindexingTaskRunner {
|
|
|
88
84
|
/**
|
|
89
85
|
* Reindexing will be triggered by the `putBatch` method.
|
|
90
86
|
*/
|
|
91
|
-
|
|
87
|
+
tableWriteBatch.put(entity.entity, {
|
|
92
88
|
...item,
|
|
89
|
+
TYPE: item.TYPE || "unknown",
|
|
93
90
|
modified: new Date().toISOString()
|
|
94
|
-
})
|
|
91
|
+
});
|
|
95
92
|
}
|
|
96
|
-
await
|
|
93
|
+
await tableWriteBatch.execute();
|
|
97
94
|
/**
|
|
98
95
|
* We always store the index settings, so we can restore them later.
|
|
99
96
|
* Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.
|
|
100
97
|
*/
|
|
101
98
|
this.keys = getKeys(results);
|
|
102
|
-
await this.manager.
|
|
99
|
+
await this.manager.controller.state.updateInput({
|
|
103
100
|
settings: this.indexManager.settings,
|
|
104
101
|
keys: this.keys
|
|
105
102
|
});
|
|
@@ -108,10 +105,10 @@ class ReindexingTaskRunner {
|
|
|
108
105
|
*/
|
|
109
106
|
if (!this.keys) {
|
|
110
107
|
await this.indexManager.enableIndexing();
|
|
111
|
-
return this.response.done("No more items to process - no last evaluated keys.");
|
|
108
|
+
return this.manager.controller.response.done("No more items to process - no last evaluated keys.");
|
|
112
109
|
}
|
|
113
110
|
}
|
|
114
|
-
return this.response.continue({
|
|
111
|
+
return this.manager.controller.response.continue({
|
|
115
112
|
keys: this.keys
|
|
116
113
|
});
|
|
117
114
|
} catch (ex) {
|
|
@@ -122,12 +119,11 @@ class ReindexingTaskRunner {
|
|
|
122
119
|
await this.indexManager.enableIndexing();
|
|
123
120
|
} catch (er) {
|
|
124
121
|
er.data = ex;
|
|
125
|
-
return this.response.error(er);
|
|
122
|
+
return this.manager.controller.response.error(er);
|
|
126
123
|
}
|
|
127
|
-
return this.response.error(ex);
|
|
124
|
+
return this.manager.controller.response.error(ex);
|
|
128
125
|
}
|
|
129
126
|
}
|
|
130
127
|
}
|
|
131
|
-
exports.ReindexingTaskRunner = ReindexingTaskRunner;
|
|
132
128
|
|
|
133
129
|
//# sourceMappingURL=ReindexingTaskRunner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["_scan","require","getKeys","results","lastEvaluatedKey","PK","SK","undefined","ReindexingTaskRunner","constructor","manager","indexManager","response","exec","keys","limit","isIndexAllowed","index","input","store","getInput","matching","includes","isCloseToTimeout","isAborted","aborted","scan","table","options","items","length","enableIndexing","done","batch","item","exists","indexExists","addInfoLog","message","entityName","_et","entity","getEntity","disableIndexing","push","putBatch","modified","Date","toISOString","write","updateInput","settings","continue","ex","er","data","error","exports"],"sources":["ReindexingTaskRunner.ts"],"sourcesContent":["import {\n IDynamoDbElasticsearchRecord,\n IElasticsearchIndexingTaskValuesKeys,\n IManager\n} from \"~/types\";\nimport { ITaskResponse, ITaskResponseResult } from \"@webiny/tasks/response/abstractions\";\nimport { scan } from \"~/helpers/scan\";\nimport { BatchWriteItem, ScanResponse } from \"@webiny/db-dynamodb\";\nimport { IndexManager } from \"~/settings\";\nimport { IIndexManager } from \"~/settings/types\";\n\nconst getKeys = (results: ScanResponse): IElasticsearchIndexingTaskValuesKeys | undefined => {\n if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {\n return {\n PK: results.lastEvaluatedKey.PK as unknown as string,\n SK: results.lastEvaluatedKey.SK as unknown as string\n };\n }\n return undefined;\n};\n\nexport class ReindexingTaskRunner {\n private readonly manager: IManager;\n private keys?: IElasticsearchIndexingTaskValuesKeys;\n\n private readonly indexManager: IIndexManager;\n private readonly response: ITaskResponse;\n\n public constructor(manager: IManager, indexManager: IndexManager) {\n this.manager = manager;\n this.response = manager.response;\n this.indexManager = indexManager;\n }\n\n /**\n * When running the task, we always must check:\n * * if task is close to timeout\n * * if task was aborted\n */\n public async exec(\n keys: IElasticsearchIndexingTaskValuesKeys | undefined = undefined,\n limit: number\n ): Promise<ITaskResponseResult> {\n this.keys = keys;\n\n const isIndexAllowed = (index: string): boolean => {\n const input = this.manager.store.getInput();\n if (typeof input.matching !== \"string\" || !input.matching) {\n return true;\n }\n return index.includes(input.matching);\n };\n\n try {\n while (this.manager.isCloseToTimeout() === false) {\n if (this.manager.isAborted()) {\n return this.response.aborted();\n }\n\n const results = await scan<IDynamoDbElasticsearchRecord>({\n table: this.manager.table,\n keys: this.keys,\n options: {\n limit\n }\n });\n if (results.items.length === 0) {\n await this.indexManager.enableIndexing();\n return this.response.done(\"No more items to process.\");\n }\n\n const batch: BatchWriteItem[] = [];\n for (const item of results.items) {\n /**\n * No index defined? Impossible but let's skip if really happens.\n */\n if (!item.index) {\n continue;\n }\n if (isIndexAllowed(item.index) === false) {\n continue;\n }\n const exists = await this.indexManager.indexExists(item.index);\n if (!exists) {\n await this.manager.store.addInfoLog({\n message: `Index \"${item.index}\" does not exist. Skipping the item.`\n });\n continue;\n }\n /**\n * Is there a possibility that entityName does not exist? What do we do at that point?\n */\n const entityName = item._et || item.entity;\n /**\n * Let's skip for now.\n */\n if (!entityName) {\n continue;\n }\n const entity = this.manager.getEntity(entityName);\n /**\n * Disable the indexing for the current index.\n * Method does nothing if indexing is already disabled.\n */\n await this.indexManager.disableIndexing(item.index);\n /**\n * Reindexing will be triggered by the `putBatch` method.\n */\n batch.push(\n entity.putBatch({\n ...item,\n modified: new Date().toISOString()\n })\n );\n }\n await this.manager.write(batch);\n /**\n * We always store the index settings, so we can restore them later.\n * Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.\n */\n this.keys = getKeys(results);\n await this.manager.store.updateInput({\n settings: this.indexManager.settings,\n keys: this.keys\n });\n /**\n * We want to make sure that, if there are no last evaluated keys, we enable indexing.\n */\n if (!this.keys) {\n await this.indexManager.enableIndexing();\n return this.response.done(\"No more items to process - no last evaluated keys.\");\n }\n }\n return this.response.continue({\n keys: this.keys\n });\n } catch (ex) {\n /**\n * We want to enable indexing if there was an error.\n */\n try {\n await this.indexManager.enableIndexing();\n } catch (er) {\n er.data = ex;\n return this.response.error(er);\n }\n return this.response.error(ex);\n }\n }\n}\n"],"mappings":";;;;;;AAMA,IAAAA,KAAA,GAAAC,OAAA;AAKA,MAAMC,OAAO,GAAIC,OAAqB,IAAuD;EACzF,IAAIA,OAAO,CAACC,gBAAgB,EAAEC,EAAE,IAAIF,OAAO,CAACC,gBAAgB,EAAEE,EAAE,EAAE;IAC9D,OAAO;MACHD,EAAE,EAAEF,OAAO,CAACC,gBAAgB,CAACC,EAAuB;MACpDC,EAAE,EAAEH,OAAO,CAACC,gBAAgB,CAACE;IACjC,CAAC;EACL;EACA,OAAOC,SAAS;AACpB,CAAC;AAEM,MAAMC,oBAAoB,CAAC;EAOvBC,WAAWA,CAACC,OAAiB,EAAEC,YAA0B,EAAE;IAC9D,IAAI,CAACD,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACE,QAAQ,GAAGF,OAAO,CAACE,QAAQ;IAChC,IAAI,CAACD,YAAY,GAAGA,YAAY;EACpC;;EAEA;AACJ;AACA;AACA;AACA;EACI,MAAaE,IAAIA,CACbC,IAAsD,GAAGP,SAAS,EAClEQ,KAAa,EACe;IAC5B,IAAI,CAACD,IAAI,GAAGA,IAAI;IAEhB,MAAME,cAAc,GAAIC,KAAa,IAAc;MAC/C,MAAMC,KAAK,GAAG,IAAI,CAACR,OAAO,CAACS,KAAK,CAACC,QAAQ,CAAC,CAAC;MAC3C,IAAI,OAAOF,KAAK,CAACG,QAAQ,KAAK,QAAQ,IAAI,CAACH,KAAK,CAACG,QAAQ,EAAE;QACvD,OAAO,IAAI;MACf;MACA,OAAOJ,KAAK,CAACK,QAAQ,CAACJ,KAAK,CAACG,QAAQ,CAAC;IACzC,CAAC;IAED,IAAI;MACA,OAAO,IAAI,CAACX,OAAO,CAACa,gBAAgB,CAAC,CAAC,KAAK,KAAK,EAAE;QAC9C,IAAI,IAAI,CAACb,OAAO,CAACc,SAAS,CAAC,CAAC,EAAE;UAC1B,OAAO,IAAI,CAACZ,QAAQ,CAACa,OAAO,CAAC,CAAC;QAClC;QAEA,MAAMtB,OAAO,GAAG,MAAM,IAAAuB,UAAI,EAA+B;UACrDC,KAAK,EAAE,IAAI,CAACjB,OAAO,CAACiB,KAAK;UACzBb,IAAI,EAAE,IAAI,CAACA,IAAI;UACfc,OAAO,EAAE;YACLb;UACJ;QACJ,CAAC,CAAC;QACF,IAAIZ,OAAO,CAAC0B,KAAK,CAACC,MAAM,KAAK,CAAC,EAAE;UAC5B,MAAM,IAAI,CAACnB,YAAY,CAACoB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACnB,QAAQ,CAACoB,IAAI,CAAC,2BAA2B,CAAC;QAC1D;QAEA,MAAMC,KAAuB,GAAG,EAAE;QAClC,KAAK,MAAMC,IAAI,IAAI/B,OAAO,CAAC0B,KAAK,EAAE;UAC9B;AACpB;AACA;UACoB,IAAI,CAACK,IAAI,CAACjB,KAAK,EAAE;YACb;UACJ;UACA,IAAID,cAAc,CAACkB,IAAI,CAACjB,KAAK,CAAC,KAAK,KAAK,EAAE;YACtC;UACJ;UACA,MAAMkB,MAAM,GAAG,MAAM,IAAI,CAACxB,YAAY,CAACyB,WAAW,CAACF,IAAI,CAACjB,KAAK,CAAC;UAC9D,IAAI,CAACkB,MAAM,EAAE;YACT,MAAM,IAAI,CAACzB,OAAO,CAACS,KAAK,CAACkB,UAAU,CAAC;cAChCC,OAAO,EAAG,UAASJ,IAAI,CAACjB,KAAM;YAClC,CAAC,CAAC;YACF;UACJ;UACA;AACpB;AACA;UACoB,MAAMsB,UAAU,GAAGL,IAAI,CAACM,GAAG,IAAIN,IAAI,CAACO,MAAM;UAC1C;AACpB;AACA;UACoB,IAAI,CAACF,UAAU,EAAE;YACb;UACJ;UACA,MAAME,MAAM,GAAG,IAAI,CAAC/B,OAAO,CAACgC,SAAS,CAACH,UAAU,CAAC;UACjD;AACpB;AACA;AACA;UACoB,MAAM,IAAI,CAAC5B,YAAY,CAACgC,eAAe,CAACT,IAAI,CAACjB,KAAK,CAAC;UACnD;AACpB;AACA;UACoBgB,KAAK,CAACW,IAAI,CACNH,MAAM,CAACI,QAAQ,CAAC;YACZ,GAAGX,IAAI;YACPY,QAAQ,EAAE,IAAIC,IAAI,CAAC,CAAC,CAACC,WAAW,CAAC;UACrC,CAAC,CACL,CAAC;QACL;QACA,MAAM,IAAI,CAACtC,OAAO,CAACuC,KAAK,CAAChB,KAAK,CAAC;QAC/B;AAChB;AACA;AACA;QACgB,IAAI,CAACnB,IAAI,GAAGZ,OAAO,CAACC,OAAO,CAAC;QAC5B,MAAM,IAAI,CAACO,OAAO,CAACS,KAAK,CAAC+B,WAAW,CAAC;UACjCC,QAAQ,EAAE,IAAI,CAACxC,YAAY,CAACwC,QAAQ;UACpCrC,IAAI,EAAE,IAAI,CAACA;QACf,CAAC,CAAC;QACF;AAChB;AACA;QACgB,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;UACZ,MAAM,IAAI,CAACH,YAAY,CAACoB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACnB,QAAQ,CAACoB,IAAI,CAAC,oDAAoD,CAAC;QACnF;MACJ;MACA,OAAO,IAAI,CAACpB,QAAQ,CAACwC,QAAQ,CAAC;QAC1BtC,IAAI,EAAE,IAAI,CAACA;MACf,CAAC,CAAC;IACN,CAAC,CAAC,OAAOuC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAI;QACA,MAAM,IAAI,CAAC1C,YAAY,CAACoB,cAAc,CAAC,CAAC;MAC5C,CAAC,CAAC,OAAOuB,EAAE,EAAE;QACTA,EAAE,CAACC,IAAI,GAAGF,EAAE;QACZ,OAAO,IAAI,CAACzC,QAAQ,CAAC4C,KAAK,CAACF,EAAE,CAAC;MAClC;MACA,OAAO,IAAI,CAAC1C,QAAQ,CAAC4C,KAAK,CAACH,EAAE,CAAC;IAClC;EACJ;AACJ;AAACI,OAAA,CAAAjD,oBAAA,GAAAA,oBAAA","ignoreList":[]}
|
|
1
|
+
{"version":3,"names":["scan","createTableWriteBatch","getKeys","results","lastEvaluatedKey","PK","SK","undefined","ReindexingTaskRunner","constructor","manager","indexManager","exec","keys","limit","isIndexAllowed","index","input","controller","state","getInput","matching","includes","runtime","isCloseToTimeout","isAborted","response","aborted","table","options","items","length","enableIndexing","done","tableWriteBatch","item","exists","indexExists","logger","info","message","entityName","_et","entity","getEntity","disableIndexing","put","TYPE","modified","Date","toISOString","execute","updateInput","settings","continue","ex","er","data","error"],"sources":["ReindexingTaskRunner.ts"],"sourcesContent":["import type {\n IDynamoDbElasticsearchRecord,\n IElasticsearchIndexingTaskValues,\n IElasticsearchIndexingTaskValuesKeys,\n IManager\n} from \"~/types.js\";\nimport type { ITaskResult } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport { scan } from \"~/helpers/scan.js\";\nimport type { ScanResponse } from \"@webiny/db-dynamodb\";\nimport { createTableWriteBatch } from \"@webiny/db-dynamodb\";\nimport type { IndexManager } from \"~/settings/index.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\n\nconst getKeys = (results: ScanResponse): IElasticsearchIndexingTaskValuesKeys | undefined => {\n if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {\n return {\n PK: results.lastEvaluatedKey.PK as unknown as string,\n SK: results.lastEvaluatedKey.SK as unknown as string\n };\n }\n return undefined;\n};\n\nexport class ReindexingTaskRunner {\n private readonly manager: IManager<IElasticsearchIndexingTaskValues>;\n private keys?: IElasticsearchIndexingTaskValuesKeys;\n private readonly indexManager: IIndexManager;\n\n public constructor(\n manager: IManager<IElasticsearchIndexingTaskValues>,\n indexManager: IndexManager\n ) {\n this.manager = manager;\n this.indexManager = indexManager;\n }\n\n /**\n * When running the task, we always must check:\n * * if task is close to timeout\n * * if task was aborted\n */\n public async exec(\n keys: IElasticsearchIndexingTaskValuesKeys | undefined = undefined,\n limit: number\n ): Promise<ITaskResult<IElasticsearchIndexingTaskValues>> {\n this.keys = keys;\n\n const isIndexAllowed = (index: string): boolean => {\n const input = this.manager.controller.state.getInput();\n if (typeof input.matching !== \"string\" || !input.matching) {\n return true;\n }\n return index.includes(input.matching);\n };\n\n try {\n while (this.manager.controller.runtime.isCloseToTimeout() === false) {\n if (this.manager.controller.runtime.isAborted()) {\n return this.manager.controller.response.aborted();\n }\n\n const results = await scan<IDynamoDbElasticsearchRecord>({\n table: this.manager.table.table,\n keys: this.keys,\n options: {\n limit\n }\n });\n if (results.items.length === 0) {\n await this.indexManager.enableIndexing();\n return this.manager.controller.response.done(\"No more items to process.\");\n }\n\n const tableWriteBatch = createTableWriteBatch({\n table: this.manager.table.table\n });\n\n for (const item of results.items) {\n /**\n * No index defined? Impossible but let's skip if really happens.\n */\n if (!item.index) {\n continue;\n }\n if (isIndexAllowed(item.index) === false) {\n continue;\n }\n const exists = await this.indexManager.indexExists(item.index);\n if (!exists) {\n await this.manager.controller.logger.info({\n message: `Index \"${item.index}\" does not exist. Skipping the item.`\n });\n continue;\n }\n /**\n * Is there a possibility that entityName does not exist? What do we do at that point?\n */\n const entityName = item._et || item.entity;\n /**\n * Let's skip for now.\n */\n if (!entityName) {\n continue;\n }\n const entity = this.manager.getEntity(entityName);\n /**\n * Disable the indexing for the current index.\n * Method does nothing if indexing is already disabled.\n */\n await this.indexManager.disableIndexing(item.index);\n /**\n * Reindexing will be triggered by the `putBatch` method.\n */\n tableWriteBatch.put(entity.entity, {\n ...item,\n TYPE: item.TYPE || \"unknown\",\n modified: new Date().toISOString()\n });\n }\n await tableWriteBatch.execute();\n /**\n * We always store the index settings, so we can restore them later.\n * Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.\n */\n this.keys = getKeys(results);\n await this.manager.controller.state.updateInput({\n settings: this.indexManager.settings,\n keys: this.keys\n });\n /**\n * We want to make sure that, if there are no last evaluated keys, we enable indexing.\n */\n if (!this.keys) {\n await this.indexManager.enableIndexing();\n return this.manager.controller.response.done(\n \"No more items to process - no last evaluated keys.\"\n );\n }\n }\n return this.manager.controller.response.continue({\n keys: this.keys\n });\n } catch (ex) {\n /**\n * We want to enable indexing if there was an error.\n */\n try {\n await this.indexManager.enableIndexing();\n } catch (er) {\n er.data = ex;\n return this.manager.controller.response.error(er);\n }\n return this.manager.controller.response.error(ex);\n }\n }\n}\n"],"mappings":"AAOA,SAASA,IAAI;AAEb,SAASC,qBAAqB,QAAQ,qBAAqB;AAI3D,MAAMC,OAAO,GAAIC,OAAqB,IAAuD;EACzF,IAAIA,OAAO,CAACC,gBAAgB,EAAEC,EAAE,IAAIF,OAAO,CAACC,gBAAgB,EAAEE,EAAE,EAAE;IAC9D,OAAO;MACHD,EAAE,EAAEF,OAAO,CAACC,gBAAgB,CAACC,EAAuB;MACpDC,EAAE,EAAEH,OAAO,CAACC,gBAAgB,CAACE;IACjC,CAAC;EACL;EACA,OAAOC,SAAS;AACpB,CAAC;AAED,OAAO,MAAMC,oBAAoB,CAAC;EAKvBC,WAAWA,CACdC,OAAmD,EACnDC,YAA0B,EAC5B;IACE,IAAI,CAACD,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,YAAY,GAAGA,YAAY;EACpC;;EAEA;AACJ;AACA;AACA;AACA;EACI,MAAaC,IAAIA,CACbC,IAAsD,GAAGN,SAAS,EAClEO,KAAa,EACyC;IACtD,IAAI,CAACD,IAAI,GAAGA,IAAI;IAEhB,MAAME,cAAc,GAAIC,KAAa,IAAc;MAC/C,MAAMC,KAAK,GAAG,IAAI,CAACP,OAAO,CAACQ,UAAU,CAACC,KAAK,CAACC,QAAQ,CAAC,CAAC;MACtD,IAAI,OAAOH,KAAK,CAACI,QAAQ,KAAK,QAAQ,IAAI,CAACJ,KAAK,CAACI,QAAQ,EAAE;QACvD,OAAO,IAAI;MACf;MACA,OAAOL,KAAK,CAACM,QAAQ,CAACL,KAAK,CAACI,QAAQ,CAAC;IACzC,CAAC;IAED,IAAI;MACA,OAAO,IAAI,CAACX,OAAO,CAACQ,UAAU,CAACK,OAAO,CAACC,gBAAgB,CAAC,CAAC,KAAK,KAAK,EAAE;QACjE,IAAI,IAAI,CAACd,OAAO,CAACQ,UAAU,CAACK,OAAO,CAACE,SAAS,CAAC,CAAC,EAAE;UAC7C,OAAO,IAAI,CAACf,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACC,OAAO,CAAC,CAAC;QACrD;QAEA,MAAMxB,OAAO,GAAG,MAAMH,IAAI,CAA+B;UACrD4B,KAAK,EAAE,IAAI,CAAClB,OAAO,CAACkB,KAAK,CAACA,KAAK;UAC/Bf,IAAI,EAAE,IAAI,CAACA,IAAI;UACfgB,OAAO,EAAE;YACLf;UACJ;QACJ,CAAC,CAAC;QACF,IAAIX,OAAO,CAAC2B,KAAK,CAACC,MAAM,KAAK,CAAC,EAAE;UAC5B,MAAM,IAAI,CAACpB,YAAY,CAACqB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACtB,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACO,IAAI,CAAC,2BAA2B,CAAC;QAC7E;QAEA,MAAMC,eAAe,GAAGjC,qBAAqB,CAAC;UAC1C2B,KAAK,EAAE,IAAI,CAAClB,OAAO,CAACkB,KAAK,CAACA;QAC9B,CAAC,CAAC;QAEF,KAAK,MAAMO,IAAI,IAAIhC,OAAO,CAAC2B,KAAK,EAAE;UAC9B;AACpB;AACA;UACoB,IAAI,CAACK,IAAI,CAACnB,KAAK,EAAE;YACb;UACJ;UACA,IAAID,cAAc,CAACoB,IAAI,CAACnB,KAAK,CAAC,KAAK,KAAK,EAAE;YACtC;UACJ;UACA,MAAMoB,MAAM,GAAG,MAAM,IAAI,CAACzB,YAAY,CAAC0B,WAAW,CAACF,IAAI,CAACnB,KAAK,CAAC;UAC9D,IAAI,CAACoB,MAAM,EAAE;YACT,MAAM,IAAI,CAAC1B,OAAO,CAACQ,UAAU,CAACoB,MAAM,CAACC,IAAI,CAAC;cACtCC,OAAO,EAAE,UAAUL,IAAI,CAACnB,KAAK;YACjC,CAAC,CAAC;YACF;UACJ;UACA;AACpB;AACA;UACoB,MAAMyB,UAAU,GAAGN,IAAI,CAACO,GAAG,IAAIP,IAAI,CAACQ,MAAM;UAC1C;AACpB;AACA;UACoB,IAAI,CAACF,UAAU,EAAE;YACb;UACJ;UACA,MAAME,MAAM,GAAG,IAAI,CAACjC,OAAO,CAACkC,SAAS,CAACH,UAAU,CAAC;UACjD;AACpB;AACA;AACA;UACoB,MAAM,IAAI,CAAC9B,YAAY,CAACkC,eAAe,CAACV,IAAI,CAACnB,KAAK,CAAC;UACnD;AACpB;AACA;UACoBkB,eAAe,CAACY,GAAG,CAACH,MAAM,CAACA,MAAM,EAAE;YAC/B,GAAGR,IAAI;YACPY,IAAI,EAAEZ,IAAI,CAACY,IAAI,IAAI,SAAS;YAC5BC,QAAQ,EAAE,IAAIC,IAAI,CAAC,CAAC,CAACC,WAAW,CAAC;UACrC,CAAC,CAAC;QACN;QACA,MAAMhB,eAAe,CAACiB,OAAO,CAAC,CAAC;QAC/B;AAChB;AACA;AACA;QACgB,IAAI,CAACtC,IAAI,GAAGX,OAAO,CAACC,OAAO,CAAC;QAC5B,MAAM,IAAI,CAACO,OAAO,CAACQ,UAAU,CAACC,KAAK,CAACiC,WAAW,CAAC;UAC5CC,QAAQ,EAAE,IAAI,CAAC1C,YAAY,CAAC0C,QAAQ;UACpCxC,IAAI,EAAE,IAAI,CAACA;QACf,CAAC,CAAC;QACF;AAChB;AACA;QACgB,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;UACZ,MAAM,IAAI,CAACF,YAAY,CAACqB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACtB,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACO,IAAI,CACxC,oDACJ,CAAC;QACL;MACJ;MACA,OAAO,IAAI,CAACvB,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAAC4B,QAAQ,CAAC;QAC7CzC,IAAI,EAAE,IAAI,CAACA;MACf,CAAC,CAAC;IACN,CAAC,CAAC,OAAO0C,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAI;QACA,MAAM,IAAI,CAAC5C,YAAY,CAACqB,cAAc,CAAC,CAAC;MAC5C,CAAC,CAAC,OAAOwB,EAAE,EAAE;QACTA,EAAE,CAACC,IAAI,GAAGF,EAAE;QACZ,OAAO,IAAI,CAAC7C,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACgC,KAAK,CAACF,EAAE,CAAC;MACrD;MACA,OAAO,IAAI,CAAC9C,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACgC,KAAK,CAACH,EAAE,CAAC;IACrD;EACJ;AACJ","ignoreList":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export * from "./reindexingTaskDefinition";
|
|
1
|
+
export * from "./reindexingTaskDefinition.js";
|
|
@@ -1,18 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
Object.defineProperty(exports, "__esModule", {
|
|
4
|
-
value: true
|
|
5
|
-
});
|
|
6
|
-
var _reindexingTaskDefinition = require("./reindexingTaskDefinition");
|
|
7
|
-
Object.keys(_reindexingTaskDefinition).forEach(function (key) {
|
|
8
|
-
if (key === "default" || key === "__esModule") return;
|
|
9
|
-
if (key in exports && exports[key] === _reindexingTaskDefinition[key]) return;
|
|
10
|
-
Object.defineProperty(exports, key, {
|
|
11
|
-
enumerable: true,
|
|
12
|
-
get: function () {
|
|
13
|
-
return _reindexingTaskDefinition[key];
|
|
14
|
-
}
|
|
15
|
-
});
|
|
16
|
-
});
|
|
1
|
+
export * from "./reindexingTaskDefinition.js";
|
|
17
2
|
|
|
18
3
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[
|
|
1
|
+
{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./reindexingTaskDefinition.js\";\n"],"mappings":"AAAA","ignoreList":[]}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { Context,
|
|
2
|
-
export declare const createElasticsearchReindexingTask: (params?: IElasticsearchTaskConfig) => import("@webiny/
|
|
1
|
+
import type { Context, IElasticsearchTaskConfig } from "../../types.js";
|
|
2
|
+
export declare const createElasticsearchReindexingTask: (params?: Partial<IElasticsearchTaskConfig>) => import("@webiny/api").ContextPlugin<Context>;
|
|
@@ -1,47 +1,48 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
isAborted,
|
|
18
|
-
store
|
|
19
|
-
}) => {
|
|
20
|
-
const {
|
|
21
|
-
Manager
|
|
22
|
-
} = await import( /* webpackChunkName: "ElasticsearchReindexingManager" */"../Manager");
|
|
23
|
-
const {
|
|
24
|
-
IndexManager
|
|
25
|
-
} = await import( /* webpackChunkName: "ElasticsearchReindexingSettings" */"../../settings");
|
|
26
|
-
const {
|
|
27
|
-
ReindexingTaskRunner
|
|
28
|
-
} = await import( /* webpackChunkName: "ElasticsearchReindexingTaskRunner" */"./ReindexingTaskRunner");
|
|
29
|
-
const manager = new Manager({
|
|
30
|
-
elasticsearchClient: params?.elasticsearchClient,
|
|
31
|
-
documentClient: params?.documentClient,
|
|
32
|
-
response,
|
|
33
|
-
context,
|
|
34
|
-
isAborted,
|
|
35
|
-
isCloseToTimeout,
|
|
36
|
-
store
|
|
37
|
-
});
|
|
38
|
-
const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});
|
|
39
|
-
const reindexing = new ReindexingTaskRunner(manager, indexManager);
|
|
40
|
-
const keys = input.keys || undefined;
|
|
41
|
-
return reindexing.exec(keys, input.limit || 100);
|
|
1
|
+
import { createContextPlugin } from "@webiny/api";
|
|
2
|
+
import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js";
|
|
3
|
+
import { getClients } from "../../helpers/getClients.js";
|
|
4
|
+
class ElasticsearchReindexingTask {
|
|
5
|
+
id = "elasticsearchReindexing";
|
|
6
|
+
title = "Elasticsearch reindexing";
|
|
7
|
+
constructor(elasticsearchClient, documentClient) {
|
|
8
|
+
this.elasticsearchClient = elasticsearchClient;
|
|
9
|
+
this.documentClient = documentClient;
|
|
10
|
+
}
|
|
11
|
+
async run({
|
|
12
|
+
input,
|
|
13
|
+
controller
|
|
14
|
+
}) {
|
|
15
|
+
if (controller.runtime.isAborted()) {
|
|
16
|
+
return controller.response.aborted();
|
|
42
17
|
}
|
|
18
|
+
const {
|
|
19
|
+
Manager
|
|
20
|
+
} = await import(/* webpackChunkName: "Manager" */"../Manager.js");
|
|
21
|
+
const {
|
|
22
|
+
IndexManager
|
|
23
|
+
} = await import(/* webpackChunkName: "IndexManager" */"../../settings/index.js");
|
|
24
|
+
const {
|
|
25
|
+
ReindexingTaskRunner
|
|
26
|
+
} = await import(/* webpackChunkName: "ReindexingTaskRunner" */"./ReindexingTaskRunner.js");
|
|
27
|
+
const manager = new Manager({
|
|
28
|
+
elasticsearchClient: this.elasticsearchClient,
|
|
29
|
+
documentClient: this.documentClient,
|
|
30
|
+
controller
|
|
31
|
+
});
|
|
32
|
+
const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});
|
|
33
|
+
const reindexing = new ReindexingTaskRunner(manager, indexManager);
|
|
34
|
+
const keys = input.keys || undefined;
|
|
35
|
+
return await reindexing.exec(keys, input.limit || 100);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export const createElasticsearchReindexingTask = params => {
|
|
39
|
+
return createContextPlugin(context => {
|
|
40
|
+
const {
|
|
41
|
+
documentClient,
|
|
42
|
+
elasticsearchClient
|
|
43
|
+
} = getClients(context, params);
|
|
44
|
+
context.container.registerFactory(TaskDefinition, () => new ElasticsearchReindexingTask(elasticsearchClient, documentClient));
|
|
43
45
|
});
|
|
44
46
|
};
|
|
45
|
-
exports.createElasticsearchReindexingTask = createElasticsearchReindexingTask;
|
|
46
47
|
|
|
47
48
|
//# sourceMappingURL=reindexingTaskDefinition.js.map
|