@webiny/api-elasticsearch-tasks 6.3.0-beta.4 → 6.4.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/abstractions/DbRegistry.js +2 -1
- package/abstractions/DbRegistry.js.map +1 -1
- package/abstractions/OpensearchTenantIndexFactory.js +2 -1
- package/abstractions/OpensearchTenantIndexFactory.js.map +1 -1
- package/abstractions/SynchronizationContext.js +2 -1
- package/abstractions/SynchronizationContext.js.map +1 -1
- package/errors/IndexSettingsGetError.js +10 -9
- package/errors/IndexSettingsGetError.js.map +1 -1
- package/errors/IndexSettingsSetError.js +10 -9
- package/errors/IndexSettingsSetError.js.map +1 -1
- package/errors/IndexingDisableError.js +3 -1
- package/errors/IndexingDisableError.js.map +1 -1
- package/errors/IndexingEnableError.js +3 -1
- package/errors/IndexingEnableError.js.map +1 -1
- package/errors/index.js +0 -2
- package/helpers/getClients.js +9 -10
- package/helpers/getClients.js.map +1 -1
- package/helpers/scan.js +12 -14
- package/helpers/scan.js.map +1 -1
- package/index.js +14 -11
- package/index.js.map +1 -1
- package/package.json +22 -22
- package/settings/DisableIndexing.js +16 -15
- package/settings/DisableIndexing.js.map +1 -1
- package/settings/EnableIndexing.js +16 -15
- package/settings/EnableIndexing.js.map +1 -1
- package/settings/IndexManager.js +67 -79
- package/settings/IndexManager.js.map +1 -1
- package/settings/IndexSettingsManager.js +31 -30
- package/settings/IndexSettingsManager.js.map +1 -1
- package/settings/index.js +0 -2
- package/settings/types.js +0 -3
- package/tasks/Manager.js +23 -24
- package/tasks/Manager.js.map +1 -1
- package/tasks/createIndexes/CreateIndexesTask.js +33 -48
- package/tasks/createIndexes/CreateIndexesTask.js.map +1 -1
- package/tasks/createIndexes/CreateIndexesTaskRunner.js +45 -61
- package/tasks/createIndexes/CreateIndexesTaskRunner.js.map +1 -1
- package/tasks/createIndexes/OnBeforeTrigger.js +28 -41
- package/tasks/createIndexes/OnBeforeTrigger.js.map +1 -1
- package/tasks/createIndexes/createIndex.js +12 -17
- package/tasks/createIndexes/createIndex.js.map +1 -1
- package/tasks/createIndexes/index.js +9 -12
- package/tasks/createIndexes/index.js.map +1 -1
- package/tasks/createIndexes/listIndexes.js +10 -16
- package/tasks/createIndexes/listIndexes.js.map +1 -1
- package/tasks/createIndexes/types.js +0 -3
- package/tasks/dataSynchronization/DataSynchronizationTask.js +52 -64
- package/tasks/dataSynchronization/DataSynchronizationTask.js.map +1 -1
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js +31 -39
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js.map +1 -1
- package/tasks/dataSynchronization/createFactories.js +4 -7
- package/tasks/dataSynchronization/createFactories.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js +65 -89
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js +49 -62
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js +50 -62
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js +0 -3
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js +0 -3
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js +7 -4
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js.map +1 -1
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.js +14 -20
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.js.map +1 -1
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js +7 -8
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js.map +1 -1
- package/tasks/dataSynchronization/entities/getTable.js +13 -19
- package/tasks/dataSynchronization/entities/getTable.js.map +1 -1
- package/tasks/dataSynchronization/entities/index.js +0 -2
- package/tasks/dataSynchronization/index.js +15 -18
- package/tasks/dataSynchronization/index.js.map +1 -1
- package/tasks/dataSynchronization/types.js +0 -3
- package/tasks/enableIndexing/EnableIndexingTaskRunner.js +27 -35
- package/tasks/enableIndexing/EnableIndexingTaskRunner.js.map +1 -1
- package/tasks/enableIndexing/index.js +23 -33
- package/tasks/enableIndexing/index.js.map +1 -1
- package/tasks/enableIndexing/types.js +0 -3
- package/tasks/index.js +0 -2
- package/tasks/reindexing/ReindexingTaskRunner.js +75 -119
- package/tasks/reindexing/ReindexingTaskRunner.js.map +1 -1
- package/tasks/reindexing/index.js +0 -2
- package/tasks/reindexing/reindexingTaskDefinition.js +25 -40
- package/tasks/reindexing/reindexingTaskDefinition.js.map +1 -1
- package/types.js +0 -3
- package/errors/index.js.map +0 -1
- package/settings/index.js.map +0 -1
- package/settings/types.js.map +0 -1
- package/tasks/createIndexes/types.js.map +0 -1
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js.map +0 -1
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js.map +0 -1
- package/tasks/dataSynchronization/entities/index.js.map +0 -1
- package/tasks/dataSynchronization/types.js.map +0 -1
- package/tasks/enableIndexing/types.js.map +0 -1
- package/tasks/index.js.map +0 -1
- package/tasks/reindexing/index.js.map +0 -1
- package/types.js.map +0 -1
|
@@ -1,41 +1,33 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
}
|
|
6
|
-
async exec(matching) {
|
|
7
|
-
if (this.manager.controller.runtime.isAborted()) {
|
|
8
|
-
return this.manager.controller.response.aborted();
|
|
1
|
+
class EnableIndexingTaskRunner {
|
|
2
|
+
constructor(manager, indexManager){
|
|
3
|
+
this.manager = manager;
|
|
4
|
+
this.indexManager = indexManager;
|
|
9
5
|
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
6
|
+
async exec(matching) {
|
|
7
|
+
if (this.manager.controller.runtime.isAborted()) return this.manager.controller.response.aborted();
|
|
8
|
+
const isIndexAllowed = (index)=>{
|
|
9
|
+
if ("string" != typeof matching || !matching) return true;
|
|
10
|
+
return index.includes(matching);
|
|
11
|
+
};
|
|
12
|
+
const indexes = await this.indexManager.list();
|
|
13
|
+
const enabled = [];
|
|
14
|
+
const failed = [];
|
|
15
|
+
for (const index of indexes)if (isIndexAllowed(index)) try {
|
|
16
|
+
await this.indexManager.enableIndexing(index);
|
|
17
|
+
enabled.push(index);
|
|
18
|
+
} catch (ex) {
|
|
19
|
+
failed.push(index);
|
|
20
|
+
await this.manager.controller.logger.error({
|
|
21
|
+
message: `Failed to enable indexing on index "${index}".`,
|
|
22
|
+
error: ex
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
return this.manager.controller.response.done("Task done.", {
|
|
26
|
+
enabled,
|
|
27
|
+
failed
|
|
31
28
|
});
|
|
32
|
-
}
|
|
33
29
|
}
|
|
34
|
-
return this.manager.controller.response.done("Task done.", {
|
|
35
|
-
enabled,
|
|
36
|
-
failed
|
|
37
|
-
});
|
|
38
|
-
}
|
|
39
30
|
}
|
|
31
|
+
export { EnableIndexingTaskRunner };
|
|
40
32
|
|
|
41
33
|
//# sourceMappingURL=EnableIndexingTaskRunner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"tasks/enableIndexing/EnableIndexingTaskRunner.js","sources":["../../../src/tasks/enableIndexing/EnableIndexingTaskRunner.ts"],"sourcesContent":["import type { IManager } from \"~/types.js\";\nimport type { ITaskResult } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport type { IndexManager } from \"~/settings/index.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { IElasticsearchEnableIndexingTaskInput } from \"~/tasks/enableIndexing/types.js\";\n\nexport class EnableIndexingTaskRunner {\n private readonly manager: IManager<IElasticsearchEnableIndexingTaskInput>;\n private readonly indexManager: IIndexManager;\n\n public constructor(\n manager: IManager<IElasticsearchEnableIndexingTaskInput>,\n indexManager: IndexManager\n ) {\n this.manager = manager;\n this.indexManager = indexManager;\n }\n\n public async exec(\n matching?: string\n ): Promise<ITaskResult<IElasticsearchEnableIndexingTaskInput>> {\n if (this.manager.controller.runtime.isAborted()) {\n return this.manager.controller.response.aborted();\n }\n\n const isIndexAllowed = (index: string): boolean => {\n if (typeof matching !== \"string\" || !matching) {\n return true;\n }\n return index.includes(matching);\n };\n\n const indexes = await this.indexManager.list();\n const enabled: string[] = [];\n const failed: string[] = [];\n for (const index of indexes) {\n if (!isIndexAllowed(index)) {\n continue;\n }\n try {\n await this.indexManager.enableIndexing(index);\n enabled.push(index);\n } catch (ex) {\n failed.push(index);\n await this.manager.controller.logger.error({\n message: `Failed to enable indexing on index \"${index}\".`,\n error: ex\n });\n }\n }\n return this.manager.controller.response.done(\"Task done.\", {\n enabled,\n failed\n });\n }\n}\n"],"names":["EnableIndexingTaskRunner","manager","indexManager","matching","isIndexAllowed","index","indexes","enabled","failed","ex"],"mappings":"AAMO,MAAMA;IAIT,YACIC,OAAwD,EACxDC,YAA0B,CAC5B;QACE,IAAI,CAAC,OAAO,GAAGD;QACf,IAAI,CAAC,YAAY,GAAGC;IACxB;IAEA,MAAa,KACTC,QAAiB,EAC0C;QAC3D,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,IACzC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO;QAGnD,MAAMC,iBAAiB,CAACC;YACpB,IAAI,AAAoB,YAApB,OAAOF,YAAyB,CAACA,UACjC,OAAO;YAEX,OAAOE,MAAM,QAAQ,CAACF;QAC1B;QAEA,MAAMG,UAAU,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI;QAC5C,MAAMC,UAAoB,EAAE;QAC5B,MAAMC,SAAmB,EAAE;QAC3B,KAAK,MAAMH,SAASC,QAChB,IAAKF,eAAeC,QAGpB,IAAI;YACA,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAACA;YACvCE,QAAQ,IAAI,CAACF;QACjB,EAAE,OAAOI,IAAI;YACTD,OAAO,IAAI,CAACH;YACZ,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC;gBACvC,SAAS,CAAC,oCAAoC,EAAEA,MAAM,EAAE,CAAC;gBACzD,OAAOI;YACX;QACJ;QAEJ,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,cAAc;YACvDF;YACAC;QACJ;IACJ;AACJ"}
|
|
@@ -5,41 +5,31 @@ import { IndexManager } from "../../settings/index.js";
|
|
|
5
5
|
import { EnableIndexingTaskRunner } from "./EnableIndexingTaskRunner.js";
|
|
6
6
|
import { getClients } from "../../helpers/getClients.js";
|
|
7
7
|
class ElasticsearchEnableIndexingTask {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
8
|
+
constructor(elasticsearchClient, documentClient){
|
|
9
|
+
this.elasticsearchClient = elasticsearchClient;
|
|
10
|
+
this.documentClient = documentClient;
|
|
11
|
+
this.id = "elasticsearchEnableIndexing";
|
|
12
|
+
this.title = "Enable Indexing on Elasticsearch Indexes";
|
|
13
|
+
}
|
|
14
|
+
async run({ input, controller }) {
|
|
15
|
+
if (controller.runtime.isAborted()) return controller.response.aborted();
|
|
16
|
+
const manager = new Manager({
|
|
17
|
+
elasticsearchClient: this.elasticsearchClient,
|
|
18
|
+
documentClient: this.documentClient,
|
|
19
|
+
controller
|
|
20
|
+
});
|
|
21
|
+
const indexManager = new IndexManager(manager.elasticsearch, {}, {
|
|
22
|
+
refreshInterval: input.refreshInterval,
|
|
23
|
+
numberOfReplicas: input.numberOfReplicas
|
|
24
|
+
});
|
|
25
|
+
const enableIndexing = new EnableIndexingTaskRunner(manager, indexManager);
|
|
26
|
+
return enableIndexing.exec(input.matching);
|
|
20
27
|
}
|
|
21
|
-
const manager = new Manager({
|
|
22
|
-
elasticsearchClient: this.elasticsearchClient,
|
|
23
|
-
documentClient: this.documentClient,
|
|
24
|
-
controller
|
|
25
|
-
});
|
|
26
|
-
const indexManager = new IndexManager(manager.elasticsearch, {}, {
|
|
27
|
-
refreshInterval: input.refreshInterval,
|
|
28
|
-
numberOfReplicas: input.numberOfReplicas
|
|
29
|
-
});
|
|
30
|
-
const enableIndexing = new EnableIndexingTaskRunner(manager, indexManager);
|
|
31
|
-
return enableIndexing.exec(input.matching);
|
|
32
|
-
}
|
|
33
28
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
// Register the task definition
|
|
39
|
-
context.container.registerFactory(TaskDefinition, () => {
|
|
40
|
-
return new ElasticsearchEnableIndexingTask(clients.elasticsearchClient, clients.documentClient);
|
|
29
|
+
const createEnableIndexingTask = (params)=>createContextPlugin((context)=>{
|
|
30
|
+
const clients = getClients(context, params);
|
|
31
|
+
context.container.registerFactory(TaskDefinition, ()=>new ElasticsearchEnableIndexingTask(clients.elasticsearchClient, clients.documentClient));
|
|
41
32
|
});
|
|
42
|
-
|
|
43
|
-
};
|
|
33
|
+
export { createEnableIndexingTask };
|
|
44
34
|
|
|
45
35
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"tasks/enableIndexing/index.js","sources":["../../../src/tasks/enableIndexing/index.ts"],"sourcesContent":["import { createContextPlugin } from \"@webiny/api\";\nimport { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport type { Context, IElasticsearchTaskConfig } from \"~/types.js\";\nimport type { IElasticsearchEnableIndexingTaskInput } from \"./types.js\";\nimport { Manager } from \"../Manager.js\";\nimport { IndexManager } from \"~/settings/index.js\";\nimport { EnableIndexingTaskRunner } from \"./EnableIndexingTaskRunner.js\";\nimport { getClients } from \"~/helpers/getClients.js\";\n\nclass ElasticsearchEnableIndexingTask implements TaskDefinition.Interface<IElasticsearchEnableIndexingTaskInput> {\n id = \"elasticsearchEnableIndexing\";\n title = \"Enable Indexing on Elasticsearch Indexes\";\n\n constructor(\n private elasticsearchClient: IElasticsearchTaskConfig[\"elasticsearchClient\"],\n private documentClient: IElasticsearchTaskConfig[\"documentClient\"]\n ) {}\n\n async run({\n input,\n controller\n }: TaskDefinition.RunParams<IElasticsearchEnableIndexingTaskInput>) {\n if (controller.runtime.isAborted()) {\n return controller.response.aborted();\n }\n\n const manager = new Manager<IElasticsearchEnableIndexingTaskInput>({\n elasticsearchClient: this.elasticsearchClient,\n documentClient: this.documentClient,\n controller\n });\n\n const indexManager = new IndexManager(\n manager.elasticsearch,\n {},\n {\n refreshInterval: input.refreshInterval,\n numberOfReplicas: input.numberOfReplicas\n }\n );\n\n const enableIndexing = new EnableIndexingTaskRunner(manager, indexManager);\n\n return enableIndexing.exec(input.matching);\n }\n}\n\nexport const createEnableIndexingTask = (params?: Partial<IElasticsearchTaskConfig>) => {\n return createContextPlugin<Context>(context => {\n const clients = getClients(context, params);\n\n // Register the task definition\n context.container.registerFactory(TaskDefinition, () => {\n return new ElasticsearchEnableIndexingTask(\n clients.elasticsearchClient,\n clients.documentClient\n );\n });\n });\n};\n"],"names":["ElasticsearchEnableIndexingTask","elasticsearchClient","documentClient","input","controller","manager","Manager","indexManager","IndexManager","enableIndexing","EnableIndexingTaskRunner","createEnableIndexingTask","params","createContextPlugin","context","clients","getClients","TaskDefinition"],"mappings":";;;;;;AASA,MAAMA;IAIF,YACYC,mBAAoE,EACpEC,cAA0D,CACpE;aAFUD,mBAAmB,GAAnBA;aACAC,cAAc,GAAdA;aALZ,EAAE,GAAG;aACL,KAAK,GAAG;IAKL;IAEH,MAAM,IAAI,EACNC,KAAK,EACLC,UAAU,EACoD,EAAE;QAChE,IAAIA,WAAW,OAAO,CAAC,SAAS,IAC5B,OAAOA,WAAW,QAAQ,CAAC,OAAO;QAGtC,MAAMC,UAAU,IAAIC,QAA+C;YAC/D,qBAAqB,IAAI,CAAC,mBAAmB;YAC7C,gBAAgB,IAAI,CAAC,cAAc;YACnCF;QACJ;QAEA,MAAMG,eAAe,IAAIC,aACrBH,QAAQ,aAAa,EACrB,CAAC,GACD;YACI,iBAAiBF,MAAM,eAAe;YACtC,kBAAkBA,MAAM,gBAAgB;QAC5C;QAGJ,MAAMM,iBAAiB,IAAIC,yBAAyBL,SAASE;QAE7D,OAAOE,eAAe,IAAI,CAACN,MAAM,QAAQ;IAC7C;AACJ;AAEO,MAAMQ,2BAA2B,CAACC,SAC9BC,oBAA6BC,CAAAA;QAChC,MAAMC,UAAUC,WAAWF,SAASF;QAGpCE,QAAQ,SAAS,CAAC,eAAe,CAACG,gBAAgB,IACvC,IAAIjB,gCACPe,QAAQ,mBAAmB,EAC3BA,QAAQ,cAAc;IAGlC"}
|
package/tasks/index.js
CHANGED
|
@@ -1,129 +1,85 @@
|
|
|
1
1
|
import { scan } from "../../helpers/scan.js";
|
|
2
2
|
import { createTableWriteBatch } from "@webiny/db-dynamodb";
|
|
3
|
-
const getKeys = results
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
SK: results.lastEvaluatedKey.SK
|
|
3
|
+
const getKeys = (results)=>{
|
|
4
|
+
if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) return {
|
|
5
|
+
PK: results.lastEvaluatedKey.PK,
|
|
6
|
+
SK: results.lastEvaluatedKey.SK
|
|
8
7
|
};
|
|
9
|
-
}
|
|
10
|
-
return undefined;
|
|
11
8
|
};
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
9
|
+
class ReindexingTaskRunner {
|
|
10
|
+
constructor(manager, indexManager){
|
|
11
|
+
this.manager = manager;
|
|
12
|
+
this.indexManager = indexManager;
|
|
13
|
+
}
|
|
14
|
+
async exec(keys, limit) {
|
|
15
|
+
this.keys = keys;
|
|
16
|
+
const isIndexAllowed = (index)=>{
|
|
17
|
+
const input = this.manager.controller.state.getInput();
|
|
18
|
+
if ("string" != typeof input.matching || !input.matching) return true;
|
|
19
|
+
return index.includes(input.matching);
|
|
20
|
+
};
|
|
21
|
+
try {
|
|
22
|
+
while(false === this.manager.controller.runtime.isCloseToTimeout()){
|
|
23
|
+
if (this.manager.controller.runtime.isAborted()) return this.manager.controller.response.aborted();
|
|
24
|
+
const results = await scan({
|
|
25
|
+
table: this.manager.table.table,
|
|
26
|
+
keys: this.keys,
|
|
27
|
+
options: {
|
|
28
|
+
limit
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
if (0 === results.items.length) {
|
|
32
|
+
await this.indexManager.enableIndexing();
|
|
33
|
+
return this.manager.controller.response.done("No more items to process.");
|
|
34
|
+
}
|
|
35
|
+
const tableWriteBatch = createTableWriteBatch({
|
|
36
|
+
table: this.manager.table.table
|
|
37
|
+
});
|
|
38
|
+
for (const item of results.items){
|
|
39
|
+
if (!item.index) continue;
|
|
40
|
+
if (false === isIndexAllowed(item.index)) continue;
|
|
41
|
+
const exists = await this.indexManager.indexExists(item.index);
|
|
42
|
+
if (!exists) {
|
|
43
|
+
await this.manager.controller.logger.info({
|
|
44
|
+
message: `Index "${item.index}" does not exist. Skipping the item.`
|
|
45
|
+
});
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const entityName = item._et || item.entity;
|
|
49
|
+
if (!entityName) continue;
|
|
50
|
+
const entity = this.manager.getEntity(entityName);
|
|
51
|
+
await this.indexManager.disableIndexing(item.index);
|
|
52
|
+
tableWriteBatch.put(entity.entity, {
|
|
53
|
+
...item,
|
|
54
|
+
TYPE: item.TYPE || "unknown",
|
|
55
|
+
modified: new Date().toISOString()
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
await tableWriteBatch.execute();
|
|
59
|
+
this.keys = getKeys(results);
|
|
60
|
+
await this.manager.controller.state.updateInput({
|
|
61
|
+
settings: this.indexManager.settings,
|
|
62
|
+
keys: this.keys
|
|
63
|
+
});
|
|
64
|
+
if (!this.keys) {
|
|
65
|
+
await this.indexManager.enableIndexing();
|
|
66
|
+
return this.manager.controller.response.done("No more items to process - no last evaluated keys.");
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return this.manager.controller.response.continue({
|
|
70
|
+
keys: this.keys
|
|
65
71
|
});
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
*/
|
|
75
|
-
if (!entityName) {
|
|
76
|
-
continue;
|
|
77
|
-
}
|
|
78
|
-
const entity = this.manager.getEntity(entityName);
|
|
79
|
-
/**
|
|
80
|
-
* Disable the indexing for the current index.
|
|
81
|
-
* Method does nothing if indexing is already disabled.
|
|
82
|
-
*/
|
|
83
|
-
await this.indexManager.disableIndexing(item.index);
|
|
84
|
-
/**
|
|
85
|
-
* Reindexing will be triggered by the `putBatch` method.
|
|
86
|
-
*/
|
|
87
|
-
tableWriteBatch.put(entity.entity, {
|
|
88
|
-
...item,
|
|
89
|
-
TYPE: item.TYPE || "unknown",
|
|
90
|
-
modified: new Date().toISOString()
|
|
91
|
-
});
|
|
92
|
-
}
|
|
93
|
-
await tableWriteBatch.execute();
|
|
94
|
-
/**
|
|
95
|
-
* We always store the index settings, so we can restore them later.
|
|
96
|
-
* Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.
|
|
97
|
-
*/
|
|
98
|
-
this.keys = getKeys(results);
|
|
99
|
-
await this.manager.controller.state.updateInput({
|
|
100
|
-
settings: this.indexManager.settings,
|
|
101
|
-
keys: this.keys
|
|
102
|
-
});
|
|
103
|
-
/**
|
|
104
|
-
* We want to make sure that, if there are no last evaluated keys, we enable indexing.
|
|
105
|
-
*/
|
|
106
|
-
if (!this.keys) {
|
|
107
|
-
await this.indexManager.enableIndexing();
|
|
108
|
-
return this.manager.controller.response.done("No more items to process - no last evaluated keys.");
|
|
72
|
+
} catch (ex) {
|
|
73
|
+
try {
|
|
74
|
+
await this.indexManager.enableIndexing();
|
|
75
|
+
} catch (er) {
|
|
76
|
+
er.data = ex;
|
|
77
|
+
return this.manager.controller.response.error(er);
|
|
78
|
+
}
|
|
79
|
+
return this.manager.controller.response.error(ex);
|
|
109
80
|
}
|
|
110
|
-
}
|
|
111
|
-
return this.manager.controller.response.continue({
|
|
112
|
-
keys: this.keys
|
|
113
|
-
});
|
|
114
|
-
} catch (ex) {
|
|
115
|
-
/**
|
|
116
|
-
* We want to enable indexing if there was an error.
|
|
117
|
-
*/
|
|
118
|
-
try {
|
|
119
|
-
await this.indexManager.enableIndexing();
|
|
120
|
-
} catch (er) {
|
|
121
|
-
er.data = ex;
|
|
122
|
-
return this.manager.controller.response.error(er);
|
|
123
|
-
}
|
|
124
|
-
return this.manager.controller.response.error(ex);
|
|
125
81
|
}
|
|
126
|
-
}
|
|
127
82
|
}
|
|
83
|
+
export { ReindexingTaskRunner };
|
|
128
84
|
|
|
129
85
|
//# sourceMappingURL=ReindexingTaskRunner.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["scan","createTableWriteBatch","getKeys","results","lastEvaluatedKey","PK","SK","undefined","ReindexingTaskRunner","constructor","manager","indexManager","exec","keys","limit","isIndexAllowed","index","input","controller","state","getInput","matching","includes","runtime","isCloseToTimeout","isAborted","response","aborted","table","options","items","length","enableIndexing","done","tableWriteBatch","item","exists","indexExists","logger","info","message","entityName","_et","entity","getEntity","disableIndexing","put","TYPE","modified","Date","toISOString","execute","updateInput","settings","continue","ex","er","data","error"],"sources":["ReindexingTaskRunner.ts"],"sourcesContent":["import type {\n IDynamoDbElasticsearchRecord,\n IElasticsearchIndexingTaskValues,\n IElasticsearchIndexingTaskValuesKeys,\n IManager\n} from \"~/types.js\";\nimport type { ITaskResult } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport { scan } from \"~/helpers/scan.js\";\nimport type { ScanResponse } from \"@webiny/db-dynamodb\";\nimport { createTableWriteBatch } from \"@webiny/db-dynamodb\";\nimport type { IndexManager } from \"~/settings/index.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\n\nconst getKeys = (results: ScanResponse): IElasticsearchIndexingTaskValuesKeys | undefined => {\n if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {\n return {\n PK: results.lastEvaluatedKey.PK as unknown as string,\n SK: results.lastEvaluatedKey.SK as unknown as string\n };\n }\n return undefined;\n};\n\nexport class ReindexingTaskRunner {\n private readonly manager: IManager<IElasticsearchIndexingTaskValues>;\n private keys?: IElasticsearchIndexingTaskValuesKeys;\n private readonly indexManager: IIndexManager;\n\n public constructor(\n manager: IManager<IElasticsearchIndexingTaskValues>,\n indexManager: IndexManager\n ) {\n this.manager = manager;\n this.indexManager = indexManager;\n }\n\n /**\n * When running the task, we always must check:\n * * if task is close to timeout\n * * if task was aborted\n */\n public async exec(\n keys: IElasticsearchIndexingTaskValuesKeys | undefined = undefined,\n limit: number\n ): Promise<ITaskResult<IElasticsearchIndexingTaskValues>> {\n this.keys = keys;\n\n const isIndexAllowed = (index: string): boolean => {\n const input = this.manager.controller.state.getInput();\n if (typeof input.matching !== \"string\" || !input.matching) {\n return true;\n }\n return index.includes(input.matching);\n };\n\n try {\n while (this.manager.controller.runtime.isCloseToTimeout() === false) {\n if (this.manager.controller.runtime.isAborted()) {\n return this.manager.controller.response.aborted();\n }\n\n const results = await scan<IDynamoDbElasticsearchRecord>({\n table: this.manager.table.table,\n keys: this.keys,\n options: {\n limit\n }\n });\n if (results.items.length === 0) {\n await this.indexManager.enableIndexing();\n return this.manager.controller.response.done(\"No more items to process.\");\n }\n\n const tableWriteBatch = createTableWriteBatch({\n table: this.manager.table.table\n });\n\n for (const item of results.items) {\n /**\n * No index defined? Impossible but let's skip if really happens.\n */\n if (!item.index) {\n continue;\n }\n if (isIndexAllowed(item.index) === false) {\n continue;\n }\n const exists = await this.indexManager.indexExists(item.index);\n if (!exists) {\n await this.manager.controller.logger.info({\n message: `Index \"${item.index}\" does not exist. Skipping the item.`\n });\n continue;\n }\n /**\n * Is there a possibility that entityName does not exist? What do we do at that point?\n */\n const entityName = item._et || item.entity;\n /**\n * Let's skip for now.\n */\n if (!entityName) {\n continue;\n }\n const entity = this.manager.getEntity(entityName);\n /**\n * Disable the indexing for the current index.\n * Method does nothing if indexing is already disabled.\n */\n await this.indexManager.disableIndexing(item.index);\n /**\n * Reindexing will be triggered by the `putBatch` method.\n */\n tableWriteBatch.put(entity.entity, {\n ...item,\n TYPE: item.TYPE || \"unknown\",\n modified: new Date().toISOString()\n });\n }\n await tableWriteBatch.execute();\n /**\n * We always store the index settings, so we can restore them later.\n * Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.\n */\n this.keys = getKeys(results);\n await this.manager.controller.state.updateInput({\n settings: this.indexManager.settings,\n keys: this.keys\n });\n /**\n * We want to make sure that, if there are no last evaluated keys, we enable indexing.\n */\n if (!this.keys) {\n await this.indexManager.enableIndexing();\n return this.manager.controller.response.done(\n \"No more items to process - no last evaluated keys.\"\n );\n }\n }\n return this.manager.controller.response.continue({\n keys: this.keys\n });\n } catch (ex) {\n /**\n * We want to enable indexing if there was an error.\n */\n try {\n await this.indexManager.enableIndexing();\n } catch (er) {\n er.data = ex;\n return this.manager.controller.response.error(er);\n }\n return this.manager.controller.response.error(ex);\n }\n }\n}\n"],"mappings":"AAOA,SAASA,IAAI;AAEb,SAASC,qBAAqB,QAAQ,qBAAqB;AAI3D,MAAMC,OAAO,GAAIC,OAAqB,IAAuD;EACzF,IAAIA,OAAO,CAACC,gBAAgB,EAAEC,EAAE,IAAIF,OAAO,CAACC,gBAAgB,EAAEE,EAAE,EAAE;IAC9D,OAAO;MACHD,EAAE,EAAEF,OAAO,CAACC,gBAAgB,CAACC,EAAuB;MACpDC,EAAE,EAAEH,OAAO,CAACC,gBAAgB,CAACE;IACjC,CAAC;EACL;EACA,OAAOC,SAAS;AACpB,CAAC;AAED,OAAO,MAAMC,oBAAoB,CAAC;EAKvBC,WAAWA,CACdC,OAAmD,EACnDC,YAA0B,EAC5B;IACE,IAAI,CAACD,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACC,YAAY,GAAGA,YAAY;EACpC;;EAEA;AACJ;AACA;AACA;AACA;EACI,MAAaC,IAAIA,CACbC,IAAsD,GAAGN,SAAS,EAClEO,KAAa,EACyC;IACtD,IAAI,CAACD,IAAI,GAAGA,IAAI;IAEhB,MAAME,cAAc,GAAIC,KAAa,IAAc;MAC/C,MAAMC,KAAK,GAAG,IAAI,CAACP,OAAO,CAACQ,UAAU,CAACC,KAAK,CAACC,QAAQ,CAAC,CAAC;MACtD,IAAI,OAAOH,KAAK,CAACI,QAAQ,KAAK,QAAQ,IAAI,CAACJ,KAAK,CAACI,QAAQ,EAAE;QACvD,OAAO,IAAI;MACf;MACA,OAAOL,KAAK,CAACM,QAAQ,CAACL,KAAK,CAACI,QAAQ,CAAC;IACzC,CAAC;IAED,IAAI;MACA,OAAO,IAAI,CAACX,OAAO,CAACQ,UAAU,CAACK,OAAO,CAACC,gBAAgB,CAAC,CAAC,KAAK,KAAK,EAAE;QACjE,IAAI,IAAI,CAACd,OAAO,CAACQ,UAAU,CAACK,OAAO,CAACE,SAAS,CAAC,CAAC,EAAE;UAC7C,OAAO,IAAI,CAACf,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACC,OAAO,CAAC,CAAC;QACrD;QAEA,MAAMxB,OAAO,GAAG,MAAMH,IAAI,CAA+B;UACrD4B,KAAK,EAAE,IAAI,CAAClB,OAAO,CAACkB,KAAK,CAACA,KAAK;UAC/Bf,IAAI,EAAE,IAAI,CAACA,IAAI;UACfgB,OAAO,EAAE;YACLf;UACJ;QACJ,CAAC,CAAC;QACF,IAAIX,OAAO,CAAC2B,KAAK,CAACC,MAAM,KAAK,CAAC,EAAE;UAC5B,MAAM,IAAI,CAACpB,YAAY,CAACqB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACtB,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACO,IAAI,CAAC,2BAA2B,CAAC;QAC7E;QAEA,MAAMC,eAAe,GAAGjC,qBAAqB,CAAC;UAC1C2B,KAAK,EAAE,IAAI,CAAClB,OAAO,CAACkB,KAAK,CAACA;QAC9B,CAAC,CAAC;QAEF,KAAK,MAAMO,IAAI,IAAIhC,OAAO,CAAC2B,KAAK,EAAE;UAC9B;AACpB;AACA;UACoB,IAAI,CAACK,IAAI,CAACnB,KAAK,EAAE;YACb;UACJ;UACA,IAAID,cAAc,CAACoB,IAAI,CAACnB,KAAK,CAAC,KAAK,KAAK,EAAE;YACtC;UACJ;UACA,MAAMoB,MAAM,GAAG,MAAM,IAAI,CAACzB,YAAY,CAAC0B,WAAW,CAACF,IAAI,CAACnB,KAAK,CAAC;UAC9D,IAAI,CAACoB,MAAM,EAAE;YACT,MAAM,IAAI,CAAC1B,OAAO,CAACQ,UAAU,CAACoB,MAAM,CAACC,IAAI,CAAC;cACtCC,OAAO,EAAE,UAAUL,IAAI,CAACnB,KAAK;YACjC,CAAC,CAAC;YACF;UACJ;UACA;AACpB;AACA;UACoB,MAAMyB,UAAU,GAAGN,IAAI,CAACO,GAAG,IAAIP,IAAI,CAACQ,MAAM;UAC1C;AACpB;AACA;UACoB,IAAI,CAACF,UAAU,EAAE;YACb;UACJ;UACA,MAAME,MAAM,GAAG,IAAI,CAACjC,OAAO,CAACkC,SAAS,CAACH,UAAU,CAAC;UACjD;AACpB;AACA;AACA;UACoB,MAAM,IAAI,CAAC9B,YAAY,CAACkC,eAAe,CAACV,IAAI,CAACnB,KAAK,CAAC;UACnD;AACpB;AACA;UACoBkB,eAAe,CAACY,GAAG,CAACH,MAAM,CAACA,MAAM,EAAE;YAC/B,GAAGR,IAAI;YACPY,IAAI,EAAEZ,IAAI,CAACY,IAAI,IAAI,SAAS;YAC5BC,QAAQ,EAAE,IAAIC,IAAI,CAAC,CAAC,CAACC,WAAW,CAAC;UACrC,CAAC,CAAC;QACN;QACA,MAAMhB,eAAe,CAACiB,OAAO,CAAC,CAAC;QAC/B;AAChB;AACA;AACA;QACgB,IAAI,CAACtC,IAAI,GAAGX,OAAO,CAACC,OAAO,CAAC;QAC5B,MAAM,IAAI,CAACO,OAAO,CAACQ,UAAU,CAACC,KAAK,CAACiC,WAAW,CAAC;UAC5CC,QAAQ,EAAE,IAAI,CAAC1C,YAAY,CAAC0C,QAAQ;UACpCxC,IAAI,EAAE,IAAI,CAACA;QACf,CAAC,CAAC;QACF;AAChB;AACA;QACgB,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;UACZ,MAAM,IAAI,CAACF,YAAY,CAACqB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACtB,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACO,IAAI,CACxC,oDACJ,CAAC;QACL;MACJ;MACA,OAAO,IAAI,CAACvB,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAAC4B,QAAQ,CAAC;QAC7CzC,IAAI,EAAE,IAAI,CAACA;MACf,CAAC,CAAC;IACN,CAAC,CAAC,OAAO0C,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAI;QACA,MAAM,IAAI,CAAC5C,YAAY,CAACqB,cAAc,CAAC,CAAC;MAC5C,CAAC,CAAC,OAAOwB,EAAE,EAAE;QACTA,EAAE,CAACC,IAAI,GAAGF,EAAE;QACZ,OAAO,IAAI,CAAC7C,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACgC,KAAK,CAACF,EAAE,CAAC;MACrD;MACA,OAAO,IAAI,CAAC9C,OAAO,CAACQ,UAAU,CAACQ,QAAQ,CAACgC,KAAK,CAACH,EAAE,CAAC;IACrD;EACJ;AACJ","ignoreList":[]}
|
|
1
|
+
{"version":3,"file":"tasks/reindexing/ReindexingTaskRunner.js","sources":["../../../src/tasks/reindexing/ReindexingTaskRunner.ts"],"sourcesContent":["import type {\n IDynamoDbElasticsearchRecord,\n IElasticsearchIndexingTaskValues,\n IElasticsearchIndexingTaskValuesKeys,\n IManager\n} from \"~/types.js\";\nimport type { ITaskResult } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport { scan } from \"~/helpers/scan.js\";\nimport type { ScanResponse } from \"@webiny/db-dynamodb\";\nimport { createTableWriteBatch } from \"@webiny/db-dynamodb\";\nimport type { IndexManager } from \"~/settings/index.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\n\nconst getKeys = (results: ScanResponse): IElasticsearchIndexingTaskValuesKeys | undefined => {\n if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {\n return {\n PK: results.lastEvaluatedKey.PK as unknown as string,\n SK: results.lastEvaluatedKey.SK as unknown as string\n };\n }\n return undefined;\n};\n\nexport class ReindexingTaskRunner {\n private readonly manager: IManager<IElasticsearchIndexingTaskValues>;\n private keys?: IElasticsearchIndexingTaskValuesKeys;\n private readonly indexManager: IIndexManager;\n\n public constructor(\n manager: IManager<IElasticsearchIndexingTaskValues>,\n indexManager: IndexManager\n ) {\n this.manager = manager;\n this.indexManager = indexManager;\n }\n\n /**\n * When running the task, we always must check:\n * * if task is close to timeout\n * * if task was aborted\n */\n public async exec(\n keys: IElasticsearchIndexingTaskValuesKeys | undefined = undefined,\n limit: number\n ): Promise<ITaskResult<IElasticsearchIndexingTaskValues>> {\n this.keys = keys;\n\n const isIndexAllowed = (index: string): boolean => {\n const input = this.manager.controller.state.getInput();\n if (typeof input.matching !== \"string\" || !input.matching) {\n return true;\n }\n return index.includes(input.matching);\n };\n\n try {\n while (this.manager.controller.runtime.isCloseToTimeout() === false) {\n if (this.manager.controller.runtime.isAborted()) {\n return this.manager.controller.response.aborted();\n }\n\n const results = await scan<IDynamoDbElasticsearchRecord>({\n table: this.manager.table.table,\n keys: this.keys,\n options: {\n limit\n }\n });\n if (results.items.length === 0) {\n await this.indexManager.enableIndexing();\n return this.manager.controller.response.done(\"No more items to process.\");\n }\n\n const tableWriteBatch = createTableWriteBatch({\n table: this.manager.table.table\n });\n\n for (const item of results.items) {\n /**\n * No index defined? Impossible but let's skip if really happens.\n */\n if (!item.index) {\n continue;\n }\n if (isIndexAllowed(item.index) === false) {\n continue;\n }\n const exists = await this.indexManager.indexExists(item.index);\n if (!exists) {\n await this.manager.controller.logger.info({\n message: `Index \"${item.index}\" does not exist. Skipping the item.`\n });\n continue;\n }\n /**\n * Is there a possibility that entityName does not exist? What do we do at that point?\n */\n const entityName = item._et || item.entity;\n /**\n * Let's skip for now.\n */\n if (!entityName) {\n continue;\n }\n const entity = this.manager.getEntity(entityName);\n /**\n * Disable the indexing for the current index.\n * Method does nothing if indexing is already disabled.\n */\n await this.indexManager.disableIndexing(item.index);\n /**\n * Reindexing will be triggered by the `putBatch` method.\n */\n tableWriteBatch.put(entity.entity, {\n ...item,\n TYPE: item.TYPE || \"unknown\",\n modified: new Date().toISOString()\n });\n }\n await tableWriteBatch.execute();\n /**\n * We always store the index settings, so we can restore them later.\n * Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.\n */\n this.keys = getKeys(results);\n await this.manager.controller.state.updateInput({\n settings: this.indexManager.settings,\n keys: this.keys\n });\n /**\n * We want to make sure that, if there are no last evaluated keys, we enable indexing.\n */\n if (!this.keys) {\n await this.indexManager.enableIndexing();\n return this.manager.controller.response.done(\n \"No more items to process - no last evaluated keys.\"\n );\n }\n }\n return this.manager.controller.response.continue({\n keys: this.keys\n });\n } catch (ex) {\n /**\n * We want to enable indexing if there was an error.\n */\n try {\n await this.indexManager.enableIndexing();\n } catch (er) {\n er.data = ex;\n return this.manager.controller.response.error(er);\n }\n return this.manager.controller.response.error(ex);\n }\n }\n}\n"],"names":["getKeys","results","ReindexingTaskRunner","manager","indexManager","keys","limit","isIndexAllowed","index","input","scan","tableWriteBatch","createTableWriteBatch","item","exists","entityName","entity","Date","ex","er"],"mappings":";;AAaA,MAAMA,UAAU,CAACC;IACb,IAAIA,QAAQ,gBAAgB,EAAE,MAAMA,QAAQ,gBAAgB,EAAE,IAC1D,OAAO;QACH,IAAIA,QAAQ,gBAAgB,CAAC,EAAE;QAC/B,IAAIA,QAAQ,gBAAgB,CAAC,EAAE;IACnC;AAGR;AAEO,MAAMC;IAKT,YACIC,OAAmD,EACnDC,YAA0B,CAC5B;QACE,IAAI,CAAC,OAAO,GAAGD;QACf,IAAI,CAAC,YAAY,GAAGC;IACxB;IAOA,MAAa,KACTC,IAAkE,EAClEC,KAAa,EACyC;QACtD,IAAI,CAAC,IAAI,GAAGD;QAEZ,MAAME,iBAAiB,CAACC;YACpB,MAAMC,QAAQ,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ;YACpD,IAAI,AAA0B,YAA1B,OAAOA,MAAM,QAAQ,IAAiB,CAACA,MAAM,QAAQ,EACrD,OAAO;YAEX,OAAOD,MAAM,QAAQ,CAACC,MAAM,QAAQ;QACxC;QAEA,IAAI;YACA,MAAO,AAAuD,UAAvD,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,gBAAgB,GAAc;gBACjE,IAAI,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,IACzC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO;gBAGnD,MAAMR,UAAU,MAAMS,KAAmC;oBACrD,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK;oBAC/B,MAAM,IAAI,CAAC,IAAI;oBACf,SAAS;wBACLJ;oBACJ;gBACJ;gBACA,IAAIL,AAAyB,MAAzBA,QAAQ,KAAK,CAAC,MAAM,EAAQ;oBAC5B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc;oBACtC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACjD;gBAEA,MAAMU,kBAAkBC,sBAAsB;oBAC1C,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK;gBACnC;gBAEA,KAAK,MAAMC,QAAQZ,QAAQ,KAAK,CAAE;oBAI9B,IAAI,CAACY,KAAK,KAAK,EACX;oBAEJ,IAAIN,AAA+B,UAA/BA,eAAeM,KAAK,KAAK,GACzB;oBAEJ,MAAMC,SAAS,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CAACD,KAAK,KAAK;oBAC7D,IAAI,CAACC,QAAQ;wBACT,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;4BACtC,SAAS,CAAC,OAAO,EAAED,KAAK,KAAK,CAAC,oCAAoC,CAAC;wBACvE;wBACA;oBACJ;oBAIA,MAAME,aAAaF,KAAK,GAAG,IAAIA,KAAK,MAAM;oBAI1C,IAAI,CAACE,YACD;oBAEJ,MAAMC,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,CAACD;oBAKtC,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CAACF,KAAK,KAAK;oBAIlDF,gBAAgB,GAAG,CAACK,OAAO,MAAM,EAAE;wBAC/B,GAAGH,IAAI;wBACP,MAAMA,KAAK,IAAI,IAAI;wBACnB,UAAU,IAAII,OAAO,WAAW;oBACpC;gBACJ;gBACA,MAAMN,gBAAgB,OAAO;gBAK7B,IAAI,CAAC,IAAI,GAAGX,QAAQC;gBACpB,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW,CAAC;oBAC5C,UAAU,IAAI,CAAC,YAAY,CAAC,QAAQ;oBACpC,MAAM,IAAI,CAAC,IAAI;gBACnB;gBAIA,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;oBACZ,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc;oBACtC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,CACxC;gBAER;YACJ;YACA,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC7C,MAAM,IAAI,CAAC,IAAI;YACnB;QACJ,EAAE,OAAOiB,IAAI;YAIT,IAAI;gBACA,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc;YAC1C,EAAE,OAAOC,IAAI;gBACTA,GAAG,IAAI,GAAGD;gBACV,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAACC;YAClD;YACA,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAACD;QAClD;IACJ;AACJ"}
|
|
@@ -2,47 +2,32 @@ import { createContextPlugin } from "@webiny/api";
|
|
|
2
2
|
import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js";
|
|
3
3
|
import { getClients } from "../../helpers/getClients.js";
|
|
4
4
|
class ElasticsearchReindexingTask {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
5
|
+
constructor(elasticsearchClient, documentClient){
|
|
6
|
+
this.elasticsearchClient = elasticsearchClient;
|
|
7
|
+
this.documentClient = documentClient;
|
|
8
|
+
this.id = "elasticsearchReindexing";
|
|
9
|
+
this.title = "Elasticsearch reindexing";
|
|
10
|
+
}
|
|
11
|
+
async run({ input, controller }) {
|
|
12
|
+
if (controller.runtime.isAborted()) return controller.response.aborted();
|
|
13
|
+
const { Manager } = await import("../Manager.js");
|
|
14
|
+
const { IndexManager } = await import("../../settings/index.js");
|
|
15
|
+
const { ReindexingTaskRunner } = await import("./ReindexingTaskRunner.js");
|
|
16
|
+
const manager = new Manager({
|
|
17
|
+
elasticsearchClient: this.elasticsearchClient,
|
|
18
|
+
documentClient: this.documentClient,
|
|
19
|
+
controller
|
|
20
|
+
});
|
|
21
|
+
const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});
|
|
22
|
+
const reindexing = new ReindexingTaskRunner(manager, indexManager);
|
|
23
|
+
const keys = input.keys || void 0;
|
|
24
|
+
return await reindexing.exec(keys, input.limit || 100);
|
|
17
25
|
}
|
|
18
|
-
const {
|
|
19
|
-
Manager
|
|
20
|
-
} = await import(/* webpackChunkName: "Manager" */"../Manager.js");
|
|
21
|
-
const {
|
|
22
|
-
IndexManager
|
|
23
|
-
} = await import(/* webpackChunkName: "IndexManager" */"../../settings/index.js");
|
|
24
|
-
const {
|
|
25
|
-
ReindexingTaskRunner
|
|
26
|
-
} = await import(/* webpackChunkName: "ReindexingTaskRunner" */"./ReindexingTaskRunner.js");
|
|
27
|
-
const manager = new Manager({
|
|
28
|
-
elasticsearchClient: this.elasticsearchClient,
|
|
29
|
-
documentClient: this.documentClient,
|
|
30
|
-
controller
|
|
31
|
-
});
|
|
32
|
-
const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});
|
|
33
|
-
const reindexing = new ReindexingTaskRunner(manager, indexManager);
|
|
34
|
-
const keys = input.keys || undefined;
|
|
35
|
-
return await reindexing.exec(keys, input.limit || 100);
|
|
36
|
-
}
|
|
37
26
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
} = getClients(context, params);
|
|
44
|
-
context.container.registerFactory(TaskDefinition, () => new ElasticsearchReindexingTask(elasticsearchClient, documentClient));
|
|
45
|
-
});
|
|
46
|
-
};
|
|
27
|
+
const createElasticsearchReindexingTask = (params)=>createContextPlugin((context)=>{
|
|
28
|
+
const { documentClient, elasticsearchClient } = getClients(context, params);
|
|
29
|
+
context.container.registerFactory(TaskDefinition, ()=>new ElasticsearchReindexingTask(elasticsearchClient, documentClient));
|
|
30
|
+
});
|
|
31
|
+
export { createElasticsearchReindexingTask };
|
|
47
32
|
|
|
48
33
|
//# sourceMappingURL=reindexingTaskDefinition.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"
|
|
1
|
+
{"version":3,"file":"tasks/reindexing/reindexingTaskDefinition.js","sources":["../../../src/tasks/reindexing/reindexingTaskDefinition.ts"],"sourcesContent":["import { createContextPlugin } from \"@webiny/api\";\nimport { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport type {\n Context,\n IElasticsearchIndexingTaskValues,\n IElasticsearchTaskConfig\n} from \"~/types.js\";\nimport { getClients } from \"~/helpers/getClients.js\";\n\nclass ElasticsearchReindexingTask implements TaskDefinition.Interface<IElasticsearchIndexingTaskValues> {\n id = \"elasticsearchReindexing\";\n title = \"Elasticsearch reindexing\";\n\n constructor(\n private elasticsearchClient: IElasticsearchTaskConfig[\"elasticsearchClient\"],\n private documentClient: IElasticsearchTaskConfig[\"documentClient\"]\n ) {}\n\n async run({ input, controller }: TaskDefinition.RunParams<IElasticsearchIndexingTaskValues>) {\n if (controller.runtime.isAborted()) {\n return controller.response.aborted();\n }\n\n const { Manager } = await import(\n /* webpackChunkName: \"Manager\" */\n \"../Manager.js\"\n );\n\n const { IndexManager } = await import(\n /* webpackChunkName: \"IndexManager\" */ \"~/settings/index.js\"\n );\n const { ReindexingTaskRunner } = await import(\n /* webpackChunkName: \"ReindexingTaskRunner\" */ \"./ReindexingTaskRunner.js\"\n );\n\n const manager = new Manager<IElasticsearchIndexingTaskValues>({\n elasticsearchClient: this.elasticsearchClient,\n documentClient: this.documentClient,\n controller\n });\n\n const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});\n const reindexing = new ReindexingTaskRunner(manager, indexManager);\n\n const keys = input.keys || undefined;\n return await reindexing.exec(keys, input.limit || 100);\n }\n}\n\nexport const createElasticsearchReindexingTask = (params?: Partial<IElasticsearchTaskConfig>) => {\n return createContextPlugin<Context>(context => {\n const { documentClient, elasticsearchClient } = getClients(context, params);\n\n context.container.registerFactory(\n TaskDefinition,\n () => new ElasticsearchReindexingTask(elasticsearchClient, documentClient)\n );\n });\n};\n"],"names":["ElasticsearchReindexingTask","elasticsearchClient","documentClient","input","controller","Manager","IndexManager","ReindexingTaskRunner","manager","indexManager","reindexing","keys","undefined","createElasticsearchReindexingTask","params","createContextPlugin","context","getClients","TaskDefinition"],"mappings":";;;AASA,MAAMA;IAIF,YACYC,mBAAoE,EACpEC,cAA0D,CACpE;aAFUD,mBAAmB,GAAnBA;aACAC,cAAc,GAAdA;aALZ,EAAE,GAAG;aACL,KAAK,GAAG;IAKL;IAEH,MAAM,IAAI,EAAEC,KAAK,EAAEC,UAAU,EAA8D,EAAE;QACzF,IAAIA,WAAW,OAAO,CAAC,SAAS,IAC5B,OAAOA,WAAW,QAAQ,CAAC,OAAO;QAGtC,MAAM,EAAEC,OAAO,EAAE,GAAG,MAAM,MAAM,CAAN;QAK1B,MAAM,EAAEC,YAAY,EAAE,GAAG,MAAM,MAAM,CAAN;QAG/B,MAAM,EAAEC,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAN;QAIvC,MAAMC,UAAU,IAAIH,QAA0C;YAC1D,qBAAqB,IAAI,CAAC,mBAAmB;YAC7C,gBAAgB,IAAI,CAAC,cAAc;YACnCD;QACJ;QAEA,MAAMK,eAAe,IAAIH,aAAaE,QAAQ,aAAa,EAAEL,MAAM,QAAQ,IAAI,CAAC;QAChF,MAAMO,aAAa,IAAIH,qBAAqBC,SAASC;QAErD,MAAME,OAAOR,MAAM,IAAI,IAAIS;QAC3B,OAAO,MAAMF,WAAW,IAAI,CAACC,MAAMR,MAAM,KAAK,IAAI;IACtD;AACJ;AAEO,MAAMU,oCAAoC,CAACC,SACvCC,oBAA6BC,CAAAA;QAChC,MAAM,EAAEd,cAAc,EAAED,mBAAmB,EAAE,GAAGgB,WAAWD,SAASF;QAEpEE,QAAQ,SAAS,CAAC,eAAe,CAC7BE,gBACA,IAAM,IAAIlB,4BAA4BC,qBAAqBC;IAEnE"}
|
package/types.js
CHANGED
package/errors/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./IndexingDisableError.js\";\nexport * from \"./IndexingEnableError.js\";\nexport * from \"./IndexSettingsGetError.js\";\nexport * from \"./IndexSettingsSetError.js\";\n"],"mappings":"AAAA;AACA;AACA;AACA","ignoreList":[]}
|
package/settings/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./IndexManager.js\";\n"],"mappings":"AAAA","ignoreList":[]}
|
package/settings/types.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { IElasticsearchIndexingTaskValuesSettings, IIndexSettingsValues } from \"~/types.js\";\n\nexport interface IIndexManager {\n settings: IElasticsearchIndexingTaskValuesSettings;\n\n list(): Promise<string[]>;\n disableIndexing(index: string): Promise<IIndexSettingsValues>;\n enableIndexing(index?: string): Promise<void>;\n createIndex(index: string, settings?: Record<string, any>): Promise<void>;\n indexExists(index: string): Promise<boolean>;\n}\n"],"mappings":"","ignoreList":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["export interface IElasticsearchCreateIndexesTaskInput {\n matching?: string;\n done?: string[];\n}\n"],"mappings":"","ignoreList":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { PrimitiveValue } from \"@webiny/api-opensearch/types.js\";\n\nexport interface IElasticsearchFetcherFetchResponseItem {\n PK: string;\n SK: string;\n _id: string;\n index: string;\n}\n\nexport interface IElasticsearchFetcherFetchParams {\n index: string;\n cursor?: PrimitiveValue[];\n limit: number;\n}\n\nexport interface IElasticsearchFetcherFetchResponse {\n done: boolean;\n totalCount: number;\n cursor?: PrimitiveValue[];\n items: IElasticsearchFetcherFetchResponseItem[];\n}\n\nexport interface IElasticsearchFetcher {\n fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;\n}\n"],"mappings":"","ignoreList":[]}
|
package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["ElasticsearchSynchronize.ts"],"sourcesContent":["export interface IElasticsearchSynchronizeExecuteParamsItem {\n PK: string;\n SK: string;\n _id: string;\n index: string;\n}\n\nexport interface IElasticsearchSynchronizeExecuteParams {\n done: boolean;\n index: string;\n items: IElasticsearchSynchronizeExecuteParamsItem[];\n}\n\nexport interface IElasticsearchSynchronizeExecuteResponse {\n done: boolean;\n}\nexport interface IElasticsearchSynchronize {\n execute(\n params: IElasticsearchSynchronizeExecuteParams\n ): Promise<IElasticsearchSynchronizeExecuteResponse>;\n}\n"],"mappings":"","ignoreList":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./getElasticsearchEntity.js\";\nexport * from \"./getElasticsearchEntityType.js\";\nexport * from \"./getTable.js\";\n"],"mappings":"AAAA;AACA;AACA","ignoreList":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { IManager } from \"~/types.js\";\nimport type { PrimitiveValue } from \"@webiny/api-opensearch/types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { IElasticsearchSynchronize } from \"~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js\";\nimport type { IElasticsearchFetcher } from \"~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js\";\nimport type { IGenericOutput } from \"@webiny/api-core/features/task/TaskService/index.js\";\nimport { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\n\nexport interface IDataSynchronizationInputValue {\n finished?: boolean;\n}\n\nexport interface IDataSynchronizationInputElasticsearchToDynamoDbValue extends IDataSynchronizationInputValue {\n index?: string;\n cursor?: PrimitiveValue[];\n}\n\nexport interface IDataSynchronizationInput {\n flow: \"elasticsearchToDynamoDb\";\n elasticsearchToDynamoDb?: IDataSynchronizationInputElasticsearchToDynamoDbValue;\n}\n\nexport type IDataSynchronizationOutput = IGenericOutput;\n\nexport type ISynchronizationRunResult =\n | TaskDefinition.ResultContinue<IDataSynchronizationInput>\n | TaskDefinition.ResultDone<IDataSynchronizationOutput>\n | TaskDefinition.ResultError\n | TaskDefinition.ResultAborted;\n\nexport interface ISynchronization {\n run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult>;\n}\n\nexport interface IElasticsearchSyncParams {\n manager: IDataSynchronizationManager;\n indexManager: IIndexManager;\n synchronize: IElasticsearchSynchronize;\n fetcher: IElasticsearchFetcher;\n}\n\nexport interface IElasticsearchSyncFactory {\n (params: IElasticsearchSyncParams): ISynchronization;\n}\n\nexport interface IFactories {\n /**\n * Delete all the records which are in the Elasticsearch but not in the Elasticsearch DynamoDB table.\n */\n elasticsearchToDynamoDb: IElasticsearchSyncFactory;\n}\n\nexport type IDataSynchronizationManager = IManager<\n IDataSynchronizationInput,\n IDataSynchronizationOutput\n>;\n"],"mappings":"","ignoreList":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["export interface IElasticsearchEnableIndexingTaskInput {\n matching?: string;\n numberOfReplicas?: number;\n refreshInterval?: string;\n}\n"],"mappings":"","ignoreList":[]}
|
package/tasks/index.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./enableIndexing/index.js\";\nexport * from \"./dataSynchronization/index.js\";\nexport * from \"./reindexing/index.js\";\nexport * from \"./createIndexes/index.js\";\n"],"mappings":"AAAA;AACA;AACA;AACA","ignoreList":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["index.ts"],"sourcesContent":["export * from \"./reindexingTaskDefinition.js\";\n"],"mappings":"AAAA","ignoreList":[]}
|