@webiny/api-elasticsearch-tasks 0.0.0-unstable.06b2ede40f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +10 -0
- package/definitions/entry.d.ts +11 -0
- package/definitions/entry.js +43 -0
- package/definitions/entry.js.map +1 -0
- package/definitions/index.d.ts +2 -0
- package/definitions/index.js +29 -0
- package/definitions/index.js.map +1 -0
- package/definitions/table.d.ts +7 -0
- package/definitions/table.js +23 -0
- package/definitions/table.js.map +1 -0
- package/errors/IndexSettingsGetError.d.ts +6 -0
- package/errors/IndexSettingsGetError.js +20 -0
- package/errors/IndexSettingsGetError.js.map +1 -0
- package/errors/IndexSettingsSetError.d.ts +6 -0
- package/errors/IndexSettingsSetError.js +20 -0
- package/errors/IndexSettingsSetError.js.map +1 -0
- package/errors/IndexingDisableError.d.ts +2 -0
- package/errors/IndexingDisableError.js +10 -0
- package/errors/IndexingDisableError.js.map +1 -0
- package/errors/IndexingEnableError.d.ts +2 -0
- package/errors/IndexingEnableError.js +10 -0
- package/errors/IndexingEnableError.js.map +1 -0
- package/errors/index.d.ts +4 -0
- package/errors/index.js +51 -0
- package/errors/index.js.map +1 -0
- package/helpers/scan.d.ts +10 -0
- package/helpers/scan.js +24 -0
- package/helpers/scan.js.map +1 -0
- package/index.d.ts +5 -0
- package/index.js +28 -0
- package/index.js.map +1 -0
- package/package.json +59 -0
- package/settings/DisableIndexing.d.ts +7 -0
- package/settings/DisableIndexing.js +27 -0
- package/settings/DisableIndexing.js.map +1 -0
- package/settings/EnableIndexing.d.ts +7 -0
- package/settings/EnableIndexing.js +27 -0
- package/settings/EnableIndexing.js.map +1 -0
- package/settings/IndexManager.d.ts +20 -0
- package/settings/IndexManager.js +100 -0
- package/settings/IndexManager.js.map +1 -0
- package/settings/IndexSettingsManager.d.ts +8 -0
- package/settings/IndexSettingsManager.js +44 -0
- package/settings/IndexSettingsManager.js.map +1 -0
- package/settings/index.d.ts +1 -0
- package/settings/index.js +18 -0
- package/settings/index.js.map +1 -0
- package/settings/types.d.ts +9 -0
- package/settings/types.js +7 -0
- package/settings/types.js.map +1 -0
- package/tasks/Manager.d.ts +33 -0
- package/tasks/Manager.js +51 -0
- package/tasks/Manager.js.map +1 -0
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.d.ts +23 -0
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.js +25 -0
- package/tasks/createIndexes/CreateElasticsearchIndexTaskPlugin.js.map +1 -0
- package/tasks/createIndexes/CreateIndexesTaskRunner.d.ts +10 -0
- package/tasks/createIndexes/CreateIndexesTaskRunner.js +77 -0
- package/tasks/createIndexes/CreateIndexesTaskRunner.js.map +1 -0
- package/tasks/createIndexes/OnBeforeTrigger.d.ts +12 -0
- package/tasks/createIndexes/OnBeforeTrigger.js +68 -0
- package/tasks/createIndexes/OnBeforeTrigger.js.map +1 -0
- package/tasks/createIndexes/createIndex.d.ts +5 -0
- package/tasks/createIndexes/createIndex.js +27 -0
- package/tasks/createIndexes/createIndex.js.map +1 -0
- package/tasks/createIndexes/index.d.ts +3 -0
- package/tasks/createIndexes/index.js +68 -0
- package/tasks/createIndexes/index.js.map +1 -0
- package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.d.ts +4 -0
- package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.js +13 -0
- package/tasks/createIndexes/listCreateElasticsearchIndexTaskPlugin.js.map +1 -0
- package/tasks/createIndexes/listIndexes.d.ts +12 -0
- package/tasks/createIndexes/listIndexes.js +53 -0
- package/tasks/createIndexes/listIndexes.js.map +1 -0
- package/tasks/createIndexes/types.d.ts +4 -0
- package/tasks/createIndexes/types.js +7 -0
- package/tasks/createIndexes/types.js.map +1 -0
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.d.ts +15 -0
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js +55 -0
- package/tasks/dataSynchronization/DataSynchronizationTaskRunner.js.map +1 -0
- package/tasks/dataSynchronization/createFactories.d.ts +2 -0
- package/tasks/dataSynchronization/createFactories.js +17 -0
- package/tasks/dataSynchronization/createFactories.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.d.ts +10 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js +98 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.d.ts +14 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js +76 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchSynchronize.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.d.ts +10 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js +77 -0
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchToDynamoDbSynchronization.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.d.ts +21 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js +7 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.d.ts +17 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js +7 -0
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js.map +1 -0
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.d.ts +2 -0
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js +13 -0
- package/tasks/dataSynchronization/elasticsearch/shouldIgnoreEsResponseError.js.map +1 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.d.ts +713 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.js +46 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntity.js.map +1 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.d.ts +11 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js +29 -0
- package/tasks/dataSynchronization/entities/getElasticsearchEntityType.js.map +1 -0
- package/tasks/dataSynchronization/entities/getTable.d.ts +7 -0
- package/tasks/dataSynchronization/entities/getTable.js +29 -0
- package/tasks/dataSynchronization/entities/getTable.js.map +1 -0
- package/tasks/dataSynchronization/entities/index.d.ts +3 -0
- package/tasks/dataSynchronization/entities/index.js +40 -0
- package/tasks/dataSynchronization/entities/index.js.map +1 -0
- package/tasks/dataSynchronization/index.d.ts +4 -0
- package/tasks/dataSynchronization/index.js +80 -0
- package/tasks/dataSynchronization/index.js.map +1 -0
- package/tasks/dataSynchronization/types.d.ts +38 -0
- package/tasks/dataSynchronization/types.js +7 -0
- package/tasks/dataSynchronization/types.js.map +1 -0
- package/tasks/enableIndexing/EnableIndexingTaskRunner.d.ts +11 -0
- package/tasks/enableIndexing/EnableIndexingTaskRunner.js +49 -0
- package/tasks/enableIndexing/EnableIndexingTaskRunner.js.map +1 -0
- package/tasks/enableIndexing/index.d.ts +3 -0
- package/tasks/enableIndexing/index.js +51 -0
- package/tasks/enableIndexing/index.js.map +1 -0
- package/tasks/enableIndexing/types.d.ts +5 -0
- package/tasks/enableIndexing/types.js +7 -0
- package/tasks/enableIndexing/types.js.map +1 -0
- package/tasks/index.d.ts +4 -0
- package/tasks/index.js +51 -0
- package/tasks/index.js.map +1 -0
- package/tasks/reindexing/ReindexingTaskRunner.d.ts +16 -0
- package/tasks/reindexing/ReindexingTaskRunner.js +137 -0
- package/tasks/reindexing/ReindexingTaskRunner.js.map +1 -0
- package/tasks/reindexing/index.d.ts +1 -0
- package/tasks/reindexing/index.js +18 -0
- package/tasks/reindexing/index.js.map +1 -0
- package/tasks/reindexing/reindexingTaskDefinition.d.ts +2 -0
- package/tasks/reindexing/reindexingTaskDefinition.js +49 -0
- package/tasks/reindexing/reindexingTaskDefinition.js.map +1 -0
- package/types.d.ts +59 -0
- package/types.js +7 -0
- package/types.js.map +1 -0
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.ReindexingTaskRunner = void 0;
|
|
7
|
+
var _scan = require("../../helpers/scan");
|
|
8
|
+
var _dbDynamodb = require("@webiny/db-dynamodb");
|
|
9
|
+
const getKeys = results => {
|
|
10
|
+
if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {
|
|
11
|
+
return {
|
|
12
|
+
PK: results.lastEvaluatedKey.PK,
|
|
13
|
+
SK: results.lastEvaluatedKey.SK
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
return undefined;
|
|
17
|
+
};
|
|
18
|
+
class ReindexingTaskRunner {
|
|
19
|
+
constructor(manager, indexManager) {
|
|
20
|
+
this.manager = manager;
|
|
21
|
+
this.response = manager.response;
|
|
22
|
+
this.indexManager = indexManager;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* When running the task, we always must check:
|
|
27
|
+
* * if task is close to timeout
|
|
28
|
+
* * if task was aborted
|
|
29
|
+
*/
|
|
30
|
+
async exec(keys = undefined, limit) {
|
|
31
|
+
this.keys = keys;
|
|
32
|
+
const isIndexAllowed = index => {
|
|
33
|
+
const input = this.manager.store.getInput();
|
|
34
|
+
if (typeof input.matching !== "string" || !input.matching) {
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
return index.includes(input.matching);
|
|
38
|
+
};
|
|
39
|
+
try {
|
|
40
|
+
while (this.manager.isCloseToTimeout() === false) {
|
|
41
|
+
if (this.manager.isAborted()) {
|
|
42
|
+
return this.response.aborted();
|
|
43
|
+
}
|
|
44
|
+
const results = await (0, _scan.scan)({
|
|
45
|
+
table: this.manager.table,
|
|
46
|
+
keys: this.keys,
|
|
47
|
+
options: {
|
|
48
|
+
limit
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
if (results.items.length === 0) {
|
|
52
|
+
await this.indexManager.enableIndexing();
|
|
53
|
+
return this.response.done("No more items to process.");
|
|
54
|
+
}
|
|
55
|
+
const tableWriteBatch = (0, _dbDynamodb.createTableWriteBatch)({
|
|
56
|
+
table: this.manager.table
|
|
57
|
+
});
|
|
58
|
+
for (const item of results.items) {
|
|
59
|
+
/**
|
|
60
|
+
* No index defined? Impossible but let's skip if really happens.
|
|
61
|
+
*/
|
|
62
|
+
if (!item.index) {
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
if (isIndexAllowed(item.index) === false) {
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
const exists = await this.indexManager.indexExists(item.index);
|
|
69
|
+
if (!exists) {
|
|
70
|
+
await this.manager.store.addInfoLog({
|
|
71
|
+
message: `Index "${item.index}" does not exist. Skipping the item.`
|
|
72
|
+
});
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Is there a possibility that entityName does not exist? What do we do at that point?
|
|
77
|
+
*/
|
|
78
|
+
const entityName = item._et || item.entity;
|
|
79
|
+
/**
|
|
80
|
+
* Let's skip for now.
|
|
81
|
+
*/
|
|
82
|
+
if (!entityName) {
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
const entity = this.manager.getEntity(entityName);
|
|
86
|
+
/**
|
|
87
|
+
* Disable the indexing for the current index.
|
|
88
|
+
* Method does nothing if indexing is already disabled.
|
|
89
|
+
*/
|
|
90
|
+
await this.indexManager.disableIndexing(item.index);
|
|
91
|
+
/**
|
|
92
|
+
* Reindexing will be triggered by the `putBatch` method.
|
|
93
|
+
*/
|
|
94
|
+
tableWriteBatch.put(entity.entity, {
|
|
95
|
+
...item,
|
|
96
|
+
TYPE: item.TYPE || "unknown",
|
|
97
|
+
modified: new Date().toISOString()
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
await tableWriteBatch.execute();
|
|
101
|
+
/**
|
|
102
|
+
* We always store the index settings, so we can restore them later.
|
|
103
|
+
* Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.
|
|
104
|
+
*/
|
|
105
|
+
this.keys = getKeys(results);
|
|
106
|
+
await this.manager.store.updateInput({
|
|
107
|
+
settings: this.indexManager.settings,
|
|
108
|
+
keys: this.keys
|
|
109
|
+
});
|
|
110
|
+
/**
|
|
111
|
+
* We want to make sure that, if there are no last evaluated keys, we enable indexing.
|
|
112
|
+
*/
|
|
113
|
+
if (!this.keys) {
|
|
114
|
+
await this.indexManager.enableIndexing();
|
|
115
|
+
return this.response.done("No more items to process - no last evaluated keys.");
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return this.response.continue({
|
|
119
|
+
keys: this.keys
|
|
120
|
+
});
|
|
121
|
+
} catch (ex) {
|
|
122
|
+
/**
|
|
123
|
+
* We want to enable indexing if there was an error.
|
|
124
|
+
*/
|
|
125
|
+
try {
|
|
126
|
+
await this.indexManager.enableIndexing();
|
|
127
|
+
} catch (er) {
|
|
128
|
+
er.data = ex;
|
|
129
|
+
return this.response.error(er);
|
|
130
|
+
}
|
|
131
|
+
return this.response.error(ex);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
exports.ReindexingTaskRunner = ReindexingTaskRunner;
|
|
136
|
+
|
|
137
|
+
//# sourceMappingURL=ReindexingTaskRunner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["_scan","require","_dbDynamodb","getKeys","results","lastEvaluatedKey","PK","SK","undefined","ReindexingTaskRunner","constructor","manager","indexManager","response","exec","keys","limit","isIndexAllowed","index","input","store","getInput","matching","includes","isCloseToTimeout","isAborted","aborted","scan","table","options","items","length","enableIndexing","done","tableWriteBatch","createTableWriteBatch","item","exists","indexExists","addInfoLog","message","entityName","_et","entity","getEntity","disableIndexing","put","TYPE","modified","Date","toISOString","execute","updateInput","settings","continue","ex","er","data","error","exports"],"sources":["ReindexingTaskRunner.ts"],"sourcesContent":["import type {\n IDynamoDbElasticsearchRecord,\n IElasticsearchIndexingTaskValues,\n IElasticsearchIndexingTaskValuesKeys,\n IManager\n} from \"~/types\";\nimport type { ITaskResponse, ITaskResponseResult } from \"@webiny/tasks/response/abstractions\";\nimport { scan } from \"~/helpers/scan\";\nimport type { ScanResponse } from \"@webiny/db-dynamodb\";\nimport { createTableWriteBatch } from \"@webiny/db-dynamodb\";\nimport type { IndexManager } from \"~/settings\";\nimport type { IIndexManager } from \"~/settings/types\";\n\nconst getKeys = (results: ScanResponse): IElasticsearchIndexingTaskValuesKeys | undefined => {\n if (results.lastEvaluatedKey?.PK && results.lastEvaluatedKey?.SK) {\n return {\n PK: results.lastEvaluatedKey.PK as unknown as string,\n SK: results.lastEvaluatedKey.SK as unknown as string\n };\n }\n return undefined;\n};\n\nexport class ReindexingTaskRunner {\n private readonly manager: IManager<IElasticsearchIndexingTaskValues>;\n private keys?: IElasticsearchIndexingTaskValuesKeys;\n\n private readonly indexManager: IIndexManager;\n private readonly response: ITaskResponse;\n\n public constructor(\n manager: IManager<IElasticsearchIndexingTaskValues>,\n indexManager: IndexManager\n ) {\n this.manager = manager;\n this.response = manager.response;\n this.indexManager = indexManager;\n }\n\n /**\n * When running the task, we always must check:\n * * if task is close to timeout\n * * if task was aborted\n */\n public async exec(\n keys: IElasticsearchIndexingTaskValuesKeys | undefined = undefined,\n limit: number\n ): Promise<ITaskResponseResult> {\n this.keys = keys;\n\n const isIndexAllowed = (index: string): boolean => {\n const input = this.manager.store.getInput();\n if (typeof input.matching !== \"string\" || !input.matching) {\n return true;\n }\n return index.includes(input.matching);\n };\n\n try {\n while (this.manager.isCloseToTimeout() === false) {\n if (this.manager.isAborted()) {\n return this.response.aborted();\n }\n\n const results = await scan<IDynamoDbElasticsearchRecord>({\n table: this.manager.table,\n keys: this.keys,\n options: {\n limit\n }\n });\n if (results.items.length === 0) {\n await this.indexManager.enableIndexing();\n return this.response.done(\"No more items to process.\");\n }\n\n const tableWriteBatch = createTableWriteBatch({\n table: this.manager.table\n });\n\n for (const item of results.items) {\n /**\n * No index defined? Impossible but let's skip if really happens.\n */\n if (!item.index) {\n continue;\n }\n if (isIndexAllowed(item.index) === false) {\n continue;\n }\n const exists = await this.indexManager.indexExists(item.index);\n if (!exists) {\n await this.manager.store.addInfoLog({\n message: `Index \"${item.index}\" does not exist. Skipping the item.`\n });\n continue;\n }\n /**\n * Is there a possibility that entityName does not exist? What do we do at that point?\n */\n const entityName = item._et || item.entity;\n /**\n * Let's skip for now.\n */\n if (!entityName) {\n continue;\n }\n const entity = this.manager.getEntity(entityName);\n /**\n * Disable the indexing for the current index.\n * Method does nothing if indexing is already disabled.\n */\n await this.indexManager.disableIndexing(item.index);\n /**\n * Reindexing will be triggered by the `putBatch` method.\n */\n tableWriteBatch.put(entity.entity, {\n ...item,\n TYPE: item.TYPE || \"unknown\",\n modified: new Date().toISOString()\n });\n }\n await tableWriteBatch.execute();\n /**\n * We always store the index settings, so we can restore them later.\n * Also, we always want to store what was the last key we processed, just in case something breaks, so we can continue from this point.\n */\n this.keys = getKeys(results);\n await this.manager.store.updateInput({\n settings: this.indexManager.settings,\n keys: this.keys\n });\n /**\n * We want to make sure that, if there are no last evaluated keys, we enable indexing.\n */\n if (!this.keys) {\n await this.indexManager.enableIndexing();\n return this.response.done(\"No more items to process - no last evaluated keys.\");\n }\n }\n return this.response.continue({\n keys: this.keys\n });\n } catch (ex) {\n /**\n * We want to enable indexing if there was an error.\n */\n try {\n await this.indexManager.enableIndexing();\n } catch (er) {\n er.data = ex;\n return this.response.error(er);\n }\n return this.response.error(ex);\n }\n }\n}\n"],"mappings":";;;;;;AAOA,IAAAA,KAAA,GAAAC,OAAA;AAEA,IAAAC,WAAA,GAAAD,OAAA;AAIA,MAAME,OAAO,GAAIC,OAAqB,IAAuD;EACzF,IAAIA,OAAO,CAACC,gBAAgB,EAAEC,EAAE,IAAIF,OAAO,CAACC,gBAAgB,EAAEE,EAAE,EAAE;IAC9D,OAAO;MACHD,EAAE,EAAEF,OAAO,CAACC,gBAAgB,CAACC,EAAuB;MACpDC,EAAE,EAAEH,OAAO,CAACC,gBAAgB,CAACE;IACjC,CAAC;EACL;EACA,OAAOC,SAAS;AACpB,CAAC;AAEM,MAAMC,oBAAoB,CAAC;EAOvBC,WAAWA,CACdC,OAAmD,EACnDC,YAA0B,EAC5B;IACE,IAAI,CAACD,OAAO,GAAGA,OAAO;IACtB,IAAI,CAACE,QAAQ,GAAGF,OAAO,CAACE,QAAQ;IAChC,IAAI,CAACD,YAAY,GAAGA,YAAY;EACpC;;EAEA;AACJ;AACA;AACA;AACA;EACI,MAAaE,IAAIA,CACbC,IAAsD,GAAGP,SAAS,EAClEQ,KAAa,EACe;IAC5B,IAAI,CAACD,IAAI,GAAGA,IAAI;IAEhB,MAAME,cAAc,GAAIC,KAAa,IAAc;MAC/C,MAAMC,KAAK,GAAG,IAAI,CAACR,OAAO,CAACS,KAAK,CAACC,QAAQ,CAAC,CAAC;MAC3C,IAAI,OAAOF,KAAK,CAACG,QAAQ,KAAK,QAAQ,IAAI,CAACH,KAAK,CAACG,QAAQ,EAAE;QACvD,OAAO,IAAI;MACf;MACA,OAAOJ,KAAK,CAACK,QAAQ,CAACJ,KAAK,CAACG,QAAQ,CAAC;IACzC,CAAC;IAED,IAAI;MACA,OAAO,IAAI,CAACX,OAAO,CAACa,gBAAgB,CAAC,CAAC,KAAK,KAAK,EAAE;QAC9C,IAAI,IAAI,CAACb,OAAO,CAACc,SAAS,CAAC,CAAC,EAAE;UAC1B,OAAO,IAAI,CAACZ,QAAQ,CAACa,OAAO,CAAC,CAAC;QAClC;QAEA,MAAMtB,OAAO,GAAG,MAAM,IAAAuB,UAAI,EAA+B;UACrDC,KAAK,EAAE,IAAI,CAACjB,OAAO,CAACiB,KAAK;UACzBb,IAAI,EAAE,IAAI,CAACA,IAAI;UACfc,OAAO,EAAE;YACLb;UACJ;QACJ,CAAC,CAAC;QACF,IAAIZ,OAAO,CAAC0B,KAAK,CAACC,MAAM,KAAK,CAAC,EAAE;UAC5B,MAAM,IAAI,CAACnB,YAAY,CAACoB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACnB,QAAQ,CAACoB,IAAI,CAAC,2BAA2B,CAAC;QAC1D;QAEA,MAAMC,eAAe,GAAG,IAAAC,iCAAqB,EAAC;UAC1CP,KAAK,EAAE,IAAI,CAACjB,OAAO,CAACiB;QACxB,CAAC,CAAC;QAEF,KAAK,MAAMQ,IAAI,IAAIhC,OAAO,CAAC0B,KAAK,EAAE;UAC9B;AACpB;AACA;UACoB,IAAI,CAACM,IAAI,CAAClB,KAAK,EAAE;YACb;UACJ;UACA,IAAID,cAAc,CAACmB,IAAI,CAAClB,KAAK,CAAC,KAAK,KAAK,EAAE;YACtC;UACJ;UACA,MAAMmB,MAAM,GAAG,MAAM,IAAI,CAACzB,YAAY,CAAC0B,WAAW,CAACF,IAAI,CAAClB,KAAK,CAAC;UAC9D,IAAI,CAACmB,MAAM,EAAE;YACT,MAAM,IAAI,CAAC1B,OAAO,CAACS,KAAK,CAACmB,UAAU,CAAC;cAChCC,OAAO,EAAE,UAAUJ,IAAI,CAAClB,KAAK;YACjC,CAAC,CAAC;YACF;UACJ;UACA;AACpB;AACA;UACoB,MAAMuB,UAAU,GAAGL,IAAI,CAACM,GAAG,IAAIN,IAAI,CAACO,MAAM;UAC1C;AACpB;AACA;UACoB,IAAI,CAACF,UAAU,EAAE;YACb;UACJ;UACA,MAAME,MAAM,GAAG,IAAI,CAAChC,OAAO,CAACiC,SAAS,CAACH,UAAU,CAAC;UACjD;AACpB;AACA;AACA;UACoB,MAAM,IAAI,CAAC7B,YAAY,CAACiC,eAAe,CAACT,IAAI,CAAClB,KAAK,CAAC;UACnD;AACpB;AACA;UACoBgB,eAAe,CAACY,GAAG,CAACH,MAAM,CAACA,MAAM,EAAE;YAC/B,GAAGP,IAAI;YACPW,IAAI,EAAEX,IAAI,CAACW,IAAI,IAAI,SAAS;YAC5BC,QAAQ,EAAE,IAAIC,IAAI,CAAC,CAAC,CAACC,WAAW,CAAC;UACrC,CAAC,CAAC;QACN;QACA,MAAMhB,eAAe,CAACiB,OAAO,CAAC,CAAC;QAC/B;AAChB;AACA;AACA;QACgB,IAAI,CAACpC,IAAI,GAAGZ,OAAO,CAACC,OAAO,CAAC;QAC5B,MAAM,IAAI,CAACO,OAAO,CAACS,KAAK,CAACgC,WAAW,CAAC;UACjCC,QAAQ,EAAE,IAAI,CAACzC,YAAY,CAACyC,QAAQ;UACpCtC,IAAI,EAAE,IAAI,CAACA;QACf,CAAC,CAAC;QACF;AAChB;AACA;QACgB,IAAI,CAAC,IAAI,CAACA,IAAI,EAAE;UACZ,MAAM,IAAI,CAACH,YAAY,CAACoB,cAAc,CAAC,CAAC;UACxC,OAAO,IAAI,CAACnB,QAAQ,CAACoB,IAAI,CAAC,oDAAoD,CAAC;QACnF;MACJ;MACA,OAAO,IAAI,CAACpB,QAAQ,CAACyC,QAAQ,CAAC;QAC1BvC,IAAI,EAAE,IAAI,CAACA;MACf,CAAC,CAAC;IACN,CAAC,CAAC,OAAOwC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAI;QACA,MAAM,IAAI,CAAC3C,YAAY,CAACoB,cAAc,CAAC,CAAC;MAC5C,CAAC,CAAC,OAAOwB,EAAE,EAAE;QACTA,EAAE,CAACC,IAAI,GAAGF,EAAE;QACZ,OAAO,IAAI,CAAC1C,QAAQ,CAAC6C,KAAK,CAACF,EAAE,CAAC;MAClC;MACA,OAAO,IAAI,CAAC3C,QAAQ,CAAC6C,KAAK,CAACH,EAAE,CAAC;IAClC;EACJ;AACJ;AAACI,OAAA,CAAAlD,oBAAA,GAAAA,oBAAA","ignoreList":[]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./reindexingTaskDefinition";
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
var _reindexingTaskDefinition = require("./reindexingTaskDefinition");
|
|
7
|
+
Object.keys(_reindexingTaskDefinition).forEach(function (key) {
|
|
8
|
+
if (key === "default" || key === "__esModule") return;
|
|
9
|
+
if (key in exports && exports[key] === _reindexingTaskDefinition[key]) return;
|
|
10
|
+
Object.defineProperty(exports, key, {
|
|
11
|
+
enumerable: true,
|
|
12
|
+
get: function () {
|
|
13
|
+
return _reindexingTaskDefinition[key];
|
|
14
|
+
}
|
|
15
|
+
});
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["_reindexingTaskDefinition","require","Object","keys","forEach","key","exports","defineProperty","enumerable","get"],"sources":["index.ts"],"sourcesContent":["export * from \"./reindexingTaskDefinition\";\n"],"mappings":";;;;;AAAA,IAAAA,yBAAA,GAAAC,OAAA;AAAAC,MAAA,CAAAC,IAAA,CAAAH,yBAAA,EAAAI,OAAA,WAAAC,GAAA;EAAA,IAAAA,GAAA,kBAAAA,GAAA;EAAA,IAAAA,GAAA,IAAAC,OAAA,IAAAA,OAAA,CAAAD,GAAA,MAAAL,yBAAA,CAAAK,GAAA;EAAAH,MAAA,CAAAK,cAAA,CAAAD,OAAA,EAAAD,GAAA;IAAAG,UAAA;IAAAC,GAAA,WAAAA,CAAA;MAAA,OAAAT,yBAAA,CAAAK,GAAA;IAAA;EAAA;AAAA","ignoreList":[]}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
import type { Context, IElasticsearchIndexingTaskValues, IElasticsearchTaskConfig } from "../../types";
|
|
2
|
+
export declare const createElasticsearchReindexingTask: (params?: IElasticsearchTaskConfig) => import("@webiny/tasks").TaskDefinitionPlugin<Context, IElasticsearchIndexingTaskValues, import("@webiny/tasks").ITaskResponseDoneResultOutput>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.createElasticsearchReindexingTask = void 0;
|
|
7
|
+
var _tasks = require("@webiny/tasks");
|
|
8
|
+
const createElasticsearchReindexingTask = params => {
|
|
9
|
+
return (0, _tasks.createTaskDefinition)({
|
|
10
|
+
id: "elasticsearchReindexing",
|
|
11
|
+
title: "Elasticsearch reindexing",
|
|
12
|
+
run: async ({
|
|
13
|
+
context,
|
|
14
|
+
isCloseToTimeout,
|
|
15
|
+
response,
|
|
16
|
+
input,
|
|
17
|
+
isAborted,
|
|
18
|
+
store,
|
|
19
|
+
timer
|
|
20
|
+
}) => {
|
|
21
|
+
const {
|
|
22
|
+
Manager
|
|
23
|
+
} = await import(/* webpackChunkName: "Manager" */"../Manager");
|
|
24
|
+
const {
|
|
25
|
+
IndexManager
|
|
26
|
+
} = await import(/* webpackChunkName: "IndexManager" */"../../settings");
|
|
27
|
+
const {
|
|
28
|
+
ReindexingTaskRunner
|
|
29
|
+
} = await import(/* webpackChunkName: "ReindexingTaskRunner" */"./ReindexingTaskRunner");
|
|
30
|
+
const manager = new Manager({
|
|
31
|
+
elasticsearchClient: params?.elasticsearchClient,
|
|
32
|
+
documentClient: params?.documentClient,
|
|
33
|
+
response,
|
|
34
|
+
context,
|
|
35
|
+
isAborted,
|
|
36
|
+
isCloseToTimeout,
|
|
37
|
+
store,
|
|
38
|
+
timer
|
|
39
|
+
});
|
|
40
|
+
const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});
|
|
41
|
+
const reindexing = new ReindexingTaskRunner(manager, indexManager);
|
|
42
|
+
const keys = input.keys || undefined;
|
|
43
|
+
return reindexing.exec(keys, input.limit || 100);
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
};
|
|
47
|
+
exports.createElasticsearchReindexingTask = createElasticsearchReindexingTask;
|
|
48
|
+
|
|
49
|
+
//# sourceMappingURL=reindexingTaskDefinition.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":["_tasks","require","createElasticsearchReindexingTask","params","createTaskDefinition","id","title","run","context","isCloseToTimeout","response","input","isAborted","store","timer","Manager","IndexManager","ReindexingTaskRunner","manager","elasticsearchClient","documentClient","indexManager","elasticsearch","settings","reindexing","keys","undefined","exec","limit","exports"],"sources":["reindexingTaskDefinition.ts"],"sourcesContent":["import { createTaskDefinition } from \"@webiny/tasks\";\nimport type { Context, IElasticsearchIndexingTaskValues, IElasticsearchTaskConfig } from \"~/types\";\n\nexport const createElasticsearchReindexingTask = (params?: IElasticsearchTaskConfig) => {\n return createTaskDefinition<Context, IElasticsearchIndexingTaskValues>({\n id: \"elasticsearchReindexing\",\n title: \"Elasticsearch reindexing\",\n run: async ({ context, isCloseToTimeout, response, input, isAborted, store, timer }) => {\n const { Manager } = await import(\n /* webpackChunkName: \"Manager\" */\n \"../Manager\"\n );\n const { IndexManager } = await import(\n /* webpackChunkName: \"IndexManager\" */ \"~/settings\"\n );\n const { ReindexingTaskRunner } = await import(\n /* webpackChunkName: \"ReindexingTaskRunner\" */ \"./ReindexingTaskRunner\"\n );\n\n const manager = new Manager<IElasticsearchIndexingTaskValues>({\n elasticsearchClient: params?.elasticsearchClient,\n documentClient: params?.documentClient,\n response,\n context,\n isAborted,\n isCloseToTimeout,\n store,\n timer\n });\n\n const indexManager = new IndexManager(manager.elasticsearch, input.settings || {});\n const reindexing = new ReindexingTaskRunner(manager, indexManager);\n\n const keys = input.keys || undefined;\n return reindexing.exec(keys, input.limit || 100);\n }\n });\n};\n"],"mappings":";;;;;;AAAA,IAAAA,MAAA,GAAAC,OAAA;AAGO,MAAMC,iCAAiC,GAAIC,MAAiC,IAAK;EACpF,OAAO,IAAAC,2BAAoB,EAA4C;IACnEC,EAAE,EAAE,yBAAyB;IAC7BC,KAAK,EAAE,0BAA0B;IACjCC,GAAG,EAAE,MAAAA,CAAO;MAAEC,OAAO;MAAEC,gBAAgB;MAAEC,QAAQ;MAAEC,KAAK;MAAEC,SAAS;MAAEC,KAAK;MAAEC;IAAM,CAAC,KAAK;MACpF,MAAM;QAAEC;MAAQ,CAAC,GAAG,MAAM,MAAM,CAC5B,6CAEJ,CAAC;MACD,MAAM;QAAEC;MAAa,CAAC,GAAG,MAAM,MAAM,CACjC,sDACJ,CAAC;MACD,MAAM;QAAEC;MAAqB,CAAC,GAAG,MAAM,MAAM,CACzC,sEACJ,CAAC;MAED,MAAMC,OAAO,GAAG,IAAIH,OAAO,CAAmC;QAC1DI,mBAAmB,EAAEhB,MAAM,EAAEgB,mBAAmB;QAChDC,cAAc,EAAEjB,MAAM,EAAEiB,cAAc;QACtCV,QAAQ;QACRF,OAAO;QACPI,SAAS;QACTH,gBAAgB;QAChBI,KAAK;QACLC;MACJ,CAAC,CAAC;MAEF,MAAMO,YAAY,GAAG,IAAIL,YAAY,CAACE,OAAO,CAACI,aAAa,EAAEX,KAAK,CAACY,QAAQ,IAAI,CAAC,CAAC,CAAC;MAClF,MAAMC,UAAU,GAAG,IAAIP,oBAAoB,CAACC,OAAO,EAAEG,YAAY,CAAC;MAElE,MAAMI,IAAI,GAAGd,KAAK,CAACc,IAAI,IAAIC,SAAS;MACpC,OAAOF,UAAU,CAACG,IAAI,CAACF,IAAI,EAAEd,KAAK,CAACiB,KAAK,IAAI,GAAG,CAAC;IACpD;EACJ,CAAC,CAAC;AACN,CAAC;AAACC,OAAA,CAAA3B,iCAAA,GAAAA,iCAAA","ignoreList":[]}
|
package/types.d.ts
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
|
|
2
|
+
import type { Context as TasksContext, IIsCloseToTimeoutCallable, ITaskManagerStore, ITaskResponse, ITaskResponseDoneResultOutput } from "@webiny/tasks/types";
|
|
3
|
+
import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb";
|
|
4
|
+
import type { Client } from "@webiny/api-elasticsearch";
|
|
5
|
+
import type { createTable } from "./definitions";
|
|
6
|
+
import type { BatchReadItem, IEntity } from "@webiny/db-dynamodb";
|
|
7
|
+
import type { ITimer } from "@webiny/handler-aws";
|
|
8
|
+
import type { GenericRecord } from "@webiny/api/types";
|
|
9
|
+
import type { Context as LoggerContext } from "@webiny/api-log/types";
|
|
10
|
+
export interface Context extends ElasticsearchContext, TasksContext, LoggerContext {
|
|
11
|
+
}
|
|
12
|
+
export interface IElasticsearchTaskConfig {
|
|
13
|
+
documentClient?: DynamoDBDocument;
|
|
14
|
+
elasticsearchClient?: Client;
|
|
15
|
+
}
|
|
16
|
+
export interface IElasticsearchIndexingTaskValuesKeys {
|
|
17
|
+
PK: string;
|
|
18
|
+
SK: string;
|
|
19
|
+
}
|
|
20
|
+
export interface IIndexSettingsValues {
|
|
21
|
+
numberOfReplicas: number;
|
|
22
|
+
refreshInterval: string;
|
|
23
|
+
}
|
|
24
|
+
export interface IElasticsearchIndexingTaskValuesSettings {
|
|
25
|
+
[key: string]: IIndexSettingsValues;
|
|
26
|
+
}
|
|
27
|
+
export interface IElasticsearchIndexingTaskValues {
|
|
28
|
+
matching?: string;
|
|
29
|
+
limit?: number;
|
|
30
|
+
keys?: IElasticsearchIndexingTaskValuesKeys;
|
|
31
|
+
settings?: IElasticsearchIndexingTaskValuesSettings;
|
|
32
|
+
}
|
|
33
|
+
export interface AugmentedError extends Error {
|
|
34
|
+
data?: GenericRecord;
|
|
35
|
+
[key: string]: any;
|
|
36
|
+
}
|
|
37
|
+
export interface IDynamoDbElasticsearchRecord {
|
|
38
|
+
PK: string;
|
|
39
|
+
SK: string;
|
|
40
|
+
TYPE?: string;
|
|
41
|
+
index: string;
|
|
42
|
+
_et?: string;
|
|
43
|
+
entity: string;
|
|
44
|
+
data: GenericRecord;
|
|
45
|
+
modified: string;
|
|
46
|
+
}
|
|
47
|
+
export interface IManager<T, O extends ITaskResponseDoneResultOutput = ITaskResponseDoneResultOutput> {
|
|
48
|
+
readonly documentClient: DynamoDBDocument;
|
|
49
|
+
readonly elasticsearch: Client;
|
|
50
|
+
readonly context: Context;
|
|
51
|
+
readonly table: ReturnType<typeof createTable>;
|
|
52
|
+
readonly isCloseToTimeout: IIsCloseToTimeoutCallable;
|
|
53
|
+
readonly isAborted: () => boolean;
|
|
54
|
+
readonly response: ITaskResponse<T, O>;
|
|
55
|
+
readonly store: ITaskManagerStore<T>;
|
|
56
|
+
readonly timer: ITimer;
|
|
57
|
+
getEntity: (name: string) => IEntity;
|
|
58
|
+
read<T>(items: BatchReadItem[]): Promise<T[]>;
|
|
59
|
+
}
|
package/types.js
ADDED
package/types.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { ElasticsearchContext } from \"@webiny/api-elasticsearch/types\";\nimport type {\n Context as TasksContext,\n IIsCloseToTimeoutCallable,\n ITaskManagerStore,\n ITaskResponse,\n ITaskResponseDoneResultOutput\n} from \"@webiny/tasks/types\";\nimport type { DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb\";\nimport type { Client } from \"@webiny/api-elasticsearch\";\nimport type { createTable } from \"~/definitions\";\nimport type { BatchReadItem, IEntity } from \"@webiny/db-dynamodb\";\nimport type { ITimer } from \"@webiny/handler-aws\";\nimport type { GenericRecord } from \"@webiny/api/types\";\nimport type { Context as LoggerContext } from \"@webiny/api-log/types\";\n\nexport interface Context extends ElasticsearchContext, TasksContext, LoggerContext {}\n\nexport interface IElasticsearchTaskConfig {\n documentClient?: DynamoDBDocument;\n elasticsearchClient?: Client;\n}\n\nexport interface IElasticsearchIndexingTaskValuesKeys {\n PK: string;\n SK: string;\n}\n\nexport interface IIndexSettingsValues {\n numberOfReplicas: number;\n refreshInterval: string;\n}\n\nexport interface IElasticsearchIndexingTaskValuesSettings {\n [key: string]: IIndexSettingsValues;\n}\n\nexport interface IElasticsearchIndexingTaskValues {\n matching?: string;\n limit?: number;\n keys?: IElasticsearchIndexingTaskValuesKeys;\n settings?: IElasticsearchIndexingTaskValuesSettings;\n}\n\nexport interface AugmentedError extends Error {\n data?: GenericRecord;\n [key: string]: any;\n}\n\nexport interface IDynamoDbElasticsearchRecord {\n PK: string;\n SK: string;\n TYPE?: string;\n index: string;\n _et?: string;\n entity: string;\n data: GenericRecord;\n modified: string;\n}\n\nexport interface IManager<\n T,\n O extends ITaskResponseDoneResultOutput = ITaskResponseDoneResultOutput\n> {\n readonly documentClient: DynamoDBDocument;\n readonly elasticsearch: Client;\n readonly context: Context;\n readonly table: ReturnType<typeof createTable>;\n readonly isCloseToTimeout: IIsCloseToTimeoutCallable;\n readonly isAborted: () => boolean;\n readonly response: ITaskResponse<T, O>;\n readonly store: ITaskManagerStore<T>;\n readonly timer: ITimer;\n\n getEntity: (name: string) => IEntity;\n\n read<T>(items: BatchReadItem[]): Promise<T[]>;\n}\n"],"mappings":"","ignoreList":[]}
|