@webiny/api-elasticsearch-tasks 6.1.0 → 6.2.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +22 -22
- package/settings/IndexManager.js +1 -1
- package/settings/IndexManager.js.map +1 -1
- package/settings/IndexSettingsManager.js +3 -3
- package/settings/IndexSettingsManager.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js +3 -2
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webiny/api-elasticsearch-tasks",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.2.0-beta.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"repository": {
|
|
@@ -13,38 +13,38 @@
|
|
|
13
13
|
],
|
|
14
14
|
"license": "MIT",
|
|
15
15
|
"dependencies": {
|
|
16
|
-
"@webiny/api": "6.
|
|
17
|
-
"@webiny/api-core": "6.
|
|
18
|
-
"@webiny/api-dynamodb-to-elasticsearch": "6.
|
|
19
|
-
"@webiny/api-opensearch": "6.
|
|
20
|
-
"@webiny/aws-sdk": "6.
|
|
21
|
-
"@webiny/db": "6.
|
|
22
|
-
"@webiny/db-dynamodb": "6.
|
|
23
|
-
"@webiny/error": "6.
|
|
24
|
-
"@webiny/feature": "6.
|
|
25
|
-
"@webiny/plugins": "6.
|
|
26
|
-
"@webiny/tasks": "6.
|
|
27
|
-
"@webiny/utils": "6.
|
|
16
|
+
"@webiny/api": "6.2.0-beta.0",
|
|
17
|
+
"@webiny/api-core": "6.2.0-beta.0",
|
|
18
|
+
"@webiny/api-dynamodb-to-elasticsearch": "6.2.0-beta.0",
|
|
19
|
+
"@webiny/api-opensearch": "6.2.0-beta.0",
|
|
20
|
+
"@webiny/aws-sdk": "6.2.0-beta.0",
|
|
21
|
+
"@webiny/db": "6.2.0-beta.0",
|
|
22
|
+
"@webiny/db-dynamodb": "6.2.0-beta.0",
|
|
23
|
+
"@webiny/error": "6.2.0-beta.0",
|
|
24
|
+
"@webiny/feature": "6.2.0-beta.0",
|
|
25
|
+
"@webiny/plugins": "6.2.0-beta.0",
|
|
26
|
+
"@webiny/tasks": "6.2.0-beta.0",
|
|
27
|
+
"@webiny/utils": "6.2.0-beta.0"
|
|
28
28
|
},
|
|
29
29
|
"devDependencies": {
|
|
30
30
|
"@webiny/api": "0.0.0",
|
|
31
|
-
"@webiny/api-headless-cms": "6.
|
|
32
|
-
"@webiny/build-tools": "6.
|
|
33
|
-
"@webiny/handler": "6.
|
|
34
|
-
"@webiny/handler-aws": "6.
|
|
35
|
-
"@webiny/handler-db": "6.
|
|
36
|
-
"@webiny/handler-graphql": "6.
|
|
37
|
-
"@webiny/project-utils": "6.
|
|
31
|
+
"@webiny/api-headless-cms": "6.2.0-beta.0",
|
|
32
|
+
"@webiny/build-tools": "6.2.0-beta.0",
|
|
33
|
+
"@webiny/handler": "6.2.0-beta.0",
|
|
34
|
+
"@webiny/handler-aws": "6.2.0-beta.0",
|
|
35
|
+
"@webiny/handler-db": "6.2.0-beta.0",
|
|
36
|
+
"@webiny/handler-graphql": "6.2.0-beta.0",
|
|
37
|
+
"@webiny/project-utils": "6.2.0-beta.0",
|
|
38
38
|
"rimraf": "6.1.3",
|
|
39
39
|
"type-fest": "5.5.0",
|
|
40
40
|
"typescript": "5.9.3",
|
|
41
|
-
"vitest": "4.1.
|
|
41
|
+
"vitest": "4.1.4"
|
|
42
42
|
},
|
|
43
43
|
"publishConfig": {
|
|
44
44
|
"access": "public",
|
|
45
45
|
"directory": "dist"
|
|
46
46
|
},
|
|
47
|
-
"gitHead": "
|
|
47
|
+
"gitHead": "3d3148358b6febbc857371930871743bec3b3939",
|
|
48
48
|
"adio": {
|
|
49
49
|
"ignore": {
|
|
50
50
|
"src": [
|
package/settings/IndexManager.js
CHANGED
|
@@ -40,7 +40,7 @@ export class IndexManager {
|
|
|
40
40
|
if (!Array.isArray(response.body)) {
|
|
41
41
|
return [];
|
|
42
42
|
}
|
|
43
|
-
return response.body.map(item => item.index).filter(filterIndex);
|
|
43
|
+
return response.body.map(item => item.index).filter(index => filterIndex(index));
|
|
44
44
|
} catch (ex) {
|
|
45
45
|
console.error(JSON.stringify({
|
|
46
46
|
message: "Failed to list indices.",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["IndexSettingsManager","DisableIndexing","EnableIndexing","getObjectProperties","defaultIndexSettings","numberOfReplicas","refreshInterval","indexPrefix","process","env","OPENSEARCH_INDEX_PREFIX","filterIndex","item","startsWith","IndexManager","settings","_settings","constructor","client","defaults","indexSettings","disable","enable","list","response","cat","indices","format","Array","isArray","body","map","index","filter","ex","console","error","JSON","stringify","message","disableIndexing","exec","enableIndexing","indexes","Object","keys","createIndex","create","indexExists","exists","ignore_unavailable","allow_no_indices","include_defaults","flat_settings","local"],"sources":["IndexManager.ts"],"sourcesContent":["import { IndexSettingsManager } from \"~/settings/IndexSettingsManager.js\";\nimport { DisableIndexing } from \"./DisableIndexing.js\";\nimport { EnableIndexing } from \"./EnableIndexing.js\";\nimport type { IElasticsearchIndexingTaskValuesSettings, IIndexSettingsValues } from \"~/types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { Client } from \"@webiny/api-opensearch\";\nimport { getObjectProperties } from \"@webiny/utils\";\n\nconst defaultIndexSettings: IIndexSettingsValues = {\n numberOfReplicas: 1,\n refreshInterval: \"1s\"\n};\n\nexport interface IListIndicesResponse {\n index: string;\n}\n\nconst indexPrefix = process.env.OPENSEARCH_INDEX_PREFIX || \"\";\nconst filterIndex = (item?: string) => {\n if (!item) {\n return false;\n } else if (item.startsWith(\".\")) {\n return false;\n } else if (indexPrefix) {\n return item.startsWith(indexPrefix);\n }\n return true;\n};\n\nexport class IndexManager implements IIndexManager {\n private readonly client: Client;\n private readonly disable: DisableIndexing;\n private readonly enable: EnableIndexing;\n private readonly _settings: IElasticsearchIndexingTaskValuesSettings;\n\n private readonly defaults: IIndexSettingsValues;\n\n public get settings(): IElasticsearchIndexingTaskValuesSettings {\n return this._settings;\n }\n\n public constructor(\n client: Client,\n settings: IElasticsearchIndexingTaskValuesSettings,\n defaults?: Partial<IIndexSettingsValues>\n ) {\n const indexSettings = new IndexSettingsManager(client);\n this.client = client;\n this.disable = new DisableIndexing(indexSettings);\n this.enable = new EnableIndexing(indexSettings);\n this._settings = settings;\n this.defaults = {\n refreshInterval: defaults?.refreshInterval || defaultIndexSettings.refreshInterval,\n numberOfReplicas: defaults?.numberOfReplicas || defaultIndexSettings.numberOfReplicas\n };\n }\n\n public async list(): Promise<string[]> {\n try {\n const response = await this.client.cat.indices
|
|
1
|
+
{"version":3,"names":["IndexSettingsManager","DisableIndexing","EnableIndexing","getObjectProperties","defaultIndexSettings","numberOfReplicas","refreshInterval","indexPrefix","process","env","OPENSEARCH_INDEX_PREFIX","filterIndex","item","startsWith","IndexManager","settings","_settings","constructor","client","defaults","indexSettings","disable","enable","list","response","cat","indices","format","Array","isArray","body","map","index","filter","ex","console","error","JSON","stringify","message","disableIndexing","exec","enableIndexing","indexes","Object","keys","createIndex","create","indexExists","exists","ignore_unavailable","allow_no_indices","include_defaults","flat_settings","local"],"sources":["IndexManager.ts"],"sourcesContent":["import { IndexSettingsManager } from \"~/settings/IndexSettingsManager.js\";\nimport { DisableIndexing } from \"./DisableIndexing.js\";\nimport { EnableIndexing } from \"./EnableIndexing.js\";\nimport type { IElasticsearchIndexingTaskValuesSettings, IIndexSettingsValues } from \"~/types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { Client } from \"@webiny/api-opensearch\";\nimport { getObjectProperties } from \"@webiny/utils\";\n\nconst defaultIndexSettings: IIndexSettingsValues = {\n numberOfReplicas: 1,\n refreshInterval: \"1s\"\n};\n\nexport interface IListIndicesResponse {\n index: string;\n}\n\nconst indexPrefix = process.env.OPENSEARCH_INDEX_PREFIX || \"\";\nconst filterIndex = (item?: string) => {\n if (!item) {\n return false;\n } else if (item.startsWith(\".\")) {\n return false;\n } else if (indexPrefix) {\n return item.startsWith(indexPrefix);\n }\n return true;\n};\n\nexport class IndexManager implements IIndexManager {\n private readonly client: Client;\n private readonly disable: DisableIndexing;\n private readonly enable: EnableIndexing;\n private readonly _settings: IElasticsearchIndexingTaskValuesSettings;\n\n private readonly defaults: IIndexSettingsValues;\n\n public get settings(): IElasticsearchIndexingTaskValuesSettings {\n return this._settings;\n }\n\n public constructor(\n client: Client,\n settings: IElasticsearchIndexingTaskValuesSettings,\n defaults?: Partial<IIndexSettingsValues>\n ) {\n const indexSettings = new IndexSettingsManager(client);\n this.client = client;\n this.disable = new DisableIndexing(indexSettings);\n this.enable = new EnableIndexing(indexSettings);\n this._settings = settings;\n this.defaults = {\n refreshInterval: defaults?.refreshInterval || defaultIndexSettings.refreshInterval,\n numberOfReplicas: defaults?.numberOfReplicas || defaultIndexSettings.numberOfReplicas\n };\n }\n\n public async list(): Promise<string[]> {\n try {\n const response = await this.client.cat.indices({\n format: \"json\"\n });\n if (!Array.isArray(response.body)) {\n return [];\n }\n return response.body\n .map(item => item.index)\n .filter((index): index is string => filterIndex(index));\n } catch (ex) {\n console.error(\n JSON.stringify({\n message: \"Failed to list indices.\",\n error: getObjectProperties(ex)\n })\n );\n return [];\n }\n }\n\n public async disableIndexing(index: string) {\n /**\n * No need to disable indexing if it's already disabled.\n */\n if (this._settings[index]) {\n return this._settings[index];\n }\n const settings = await this.disable.exec(index);\n this._settings[index] = settings;\n return settings;\n }\n\n public async enableIndexing(index?: string) {\n if (!index) {\n const indexes = Object.keys(this._settings);\n for (const index of indexes) {\n await this.enableIndexing(index);\n }\n return;\n }\n const settings = this._settings[index] || this.defaults;\n await this.enable.exec(index, settings);\n }\n\n public async createIndex(index: string, settings?: Record<string, any>): Promise<void> {\n await this.client.indices.create({\n index,\n body: settings\n });\n }\n\n public async indexExists(index: string): Promise<boolean> {\n const response = await this.client.indices.exists({\n index,\n ignore_unavailable: false,\n allow_no_indices: true,\n include_defaults: true,\n flat_settings: false,\n local: false\n });\n return !!response.body;\n }\n}\n"],"mappings":"AAAA,SAASA,oBAAoB;AAC7B,SAASC,eAAe;AACxB,SAASC,cAAc;AAIvB,SAASC,mBAAmB,QAAQ,eAAe;AAEnD,MAAMC,oBAA0C,GAAG;EAC/CC,gBAAgB,EAAE,CAAC;EACnBC,eAAe,EAAE;AACrB,CAAC;AAMD,MAAMC,WAAW,GAAGC,OAAO,CAACC,GAAG,CAACC,uBAAuB,IAAI,EAAE;AAC7D,MAAMC,WAAW,GAAIC,IAAa,IAAK;EACnC,IAAI,CAACA,IAAI,EAAE;IACP,OAAO,KAAK;EAChB,CAAC,MAAM,IAAIA,IAAI,CAACC,UAAU,CAAC,GAAG,CAAC,EAAE;IAC7B,OAAO,KAAK;EAChB,CAAC,MAAM,IAAIN,WAAW,EAAE;IACpB,OAAOK,IAAI,CAACC,UAAU,CAACN,WAAW,CAAC;EACvC;EACA,OAAO,IAAI;AACf,CAAC;AAED,OAAO,MAAMO,YAAY,CAA0B;EAQ/C,IAAWC,QAAQA,CAAA,EAA6C;IAC5D,OAAO,IAAI,CAACC,SAAS;EACzB;EAEOC,WAAWA,CACdC,MAAc,EACdH,QAAkD,EAClDI,QAAwC,EAC1C;IACE,MAAMC,aAAa,GAAG,IAAIpB,oBAAoB,CAACkB,MAAM,CAAC;IACtD,IAAI,CAACA,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACG,OAAO,GAAG,IAAIpB,eAAe,CAACmB,aAAa,CAAC;IACjD,IAAI,CAACE,MAAM,GAAG,IAAIpB,cAAc,CAACkB,aAAa,CAAC;IAC/C,IAAI,CAACJ,SAAS,GAAGD,QAAQ;IACzB,IAAI,CAACI,QAAQ,GAAG;MACZb,eAAe,EAAEa,QAAQ,EAAEb,eAAe,IAAIF,oBAAoB,CAACE,eAAe;MAClFD,gBAAgB,EAAEc,QAAQ,EAAEd,gBAAgB,IAAID,oBAAoB,CAACC;IACzE,CAAC;EACL;EAEA,MAAakB,IAAIA,CAAA,EAAsB;IACnC,IAAI;MACA,MAAMC,QAAQ,GAAG,MAAM,IAAI,CAACN,MAAM,CAACO,GAAG,CAACC,OAAO,CAAC;QAC3CC,MAAM,EAAE;MACZ,CAAC,CAAC;MACF,IAAI,CAACC,KAAK,CAACC,OAAO,CAACL,QAAQ,CAACM,IAAI,CAAC,EAAE;QAC/B,OAAO,EAAE;MACb;MACA,OAAON,QAAQ,CAACM,IAAI,CACfC,GAAG,CAACnB,IAAI,IAAIA,IAAI,CAACoB,KAAK,CAAC,CACvBC,MAAM,CAAED,KAAK,IAAsBrB,WAAW,CAACqB,KAAK,CAAC,CAAC;IAC/D,CAAC,CAAC,OAAOE,EAAE,EAAE;MACTC,OAAO,CAACC,KAAK,CACTC,IAAI,CAACC,SAAS,CAAC;QACXC,OAAO,EAAE,yBAAyB;QAClCH,KAAK,EAAEjC,mBAAmB,CAAC+B,EAAE;MACjC,CAAC,CACL,CAAC;MACD,OAAO,EAAE;IACb;EACJ;EAEA,MAAaM,eAAeA,CAACR,KAAa,EAAE;IACxC;AACR;AACA;IACQ,IAAI,IAAI,CAAChB,SAAS,CAACgB,KAAK,CAAC,EAAE;MACvB,OAAO,IAAI,CAAChB,SAAS,CAACgB,KAAK,CAAC;IAChC;IACA,MAAMjB,QAAQ,GAAG,MAAM,IAAI,CAACM,OAAO,CAACoB,IAAI,CAACT,KAAK,CAAC;IAC/C,IAAI,CAAChB,SAAS,CAACgB,KAAK,CAAC,GAAGjB,QAAQ;IAChC,OAAOA,QAAQ;EACnB;EAEA,MAAa2B,cAAcA,CAACV,KAAc,EAAE;IACxC,IAAI,CAACA,KAAK,EAAE;MACR,MAAMW,OAAO,GAAGC,MAAM,CAACC,IAAI,CAAC,IAAI,CAAC7B,SAAS,CAAC;MAC3C,KAAK,MAAMgB,KAAK,IAAIW,OAAO,EAAE;QACzB,MAAM,IAAI,CAACD,cAAc,CAACV,KAAK,CAAC;MACpC;MACA;IACJ;IACA,MAAMjB,QAAQ,GAAG,IAAI,CAACC,SAAS,CAACgB,KAAK,CAAC,IAAI,IAAI,CAACb,QAAQ;IACvD,MAAM,IAAI,CAACG,MAAM,CAACmB,IAAI,CAACT,KAAK,EAAEjB,QAAQ,CAAC;EAC3C;EAEA,MAAa+B,WAAWA,CAACd,KAAa,EAAEjB,QAA8B,EAAiB;IACnF,MAAM,IAAI,CAACG,MAAM,CAACQ,OAAO,CAACqB,MAAM,CAAC;MAC7Bf,KAAK;MACLF,IAAI,EAAEf;IACV,CAAC,CAAC;EACN;EAEA,MAAaiC,WAAWA,CAAChB,KAAa,EAAoB;IACtD,MAAMR,QAAQ,GAAG,MAAM,IAAI,CAACN,MAAM,CAACQ,OAAO,CAACuB,MAAM,CAAC;MAC9CjB,KAAK;MACLkB,kBAAkB,EAAE,KAAK;MACzBC,gBAAgB,EAAE,IAAI;MACtBC,gBAAgB,EAAE,IAAI;MACtBC,aAAa,EAAE,KAAK;MACpBC,KAAK,EAAE;IACX,CAAC,CAAC;IACF,OAAO,CAAC,CAAC9B,QAAQ,CAACM,IAAI;EAC1B;AACJ","ignoreList":[]}
|
|
@@ -8,10 +8,10 @@ export class IndexSettingsManager {
|
|
|
8
8
|
const response = await this.elasticsearch.indices.getSettings({
|
|
9
9
|
index
|
|
10
10
|
});
|
|
11
|
-
const setting = response.body[index]
|
|
11
|
+
const setting = response.body[index]?.settings?.index;
|
|
12
12
|
return {
|
|
13
|
-
numberOfReplicas: setting
|
|
14
|
-
refreshInterval: setting
|
|
13
|
+
numberOfReplicas: parseInt(String(setting?.number_of_replicas ?? "0"), 10),
|
|
14
|
+
refreshInterval: setting?.refresh_interval ?? "1s"
|
|
15
15
|
};
|
|
16
16
|
} catch (ex) {
|
|
17
17
|
throw new IndexSettingsGetError(ex, index);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["IndexSettingsGetError","IndexSettingsSetError","IndexSettingsManager","constructor","elasticsearch","getSettings","index","response","indices","setting","body","settings","numberOfReplicas","number_of_replicas","refreshInterval","refresh_interval","ex","setSettings","putSettings"],"sources":["IndexSettingsManager.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-opensearch\";\nimport { IndexSettingsGetError, IndexSettingsSetError } from \"~/errors/index.js\";\nimport type { IIndexSettingsValues } from \"~/types.js\";\n\nexport class IndexSettingsManager {\n private readonly elasticsearch: Client;\n\n public constructor(elasticsearch: Client) {\n this.elasticsearch = elasticsearch;\n }\n\n public async getSettings(index: string): Promise<IIndexSettingsValues> {\n try {\n const response = await this.elasticsearch.indices.getSettings({\n index\n });\n\n const setting = response.body[index]
|
|
1
|
+
{"version":3,"names":["IndexSettingsGetError","IndexSettingsSetError","IndexSettingsManager","constructor","elasticsearch","getSettings","index","response","indices","setting","body","settings","numberOfReplicas","parseInt","String","number_of_replicas","refreshInterval","refresh_interval","ex","setSettings","putSettings"],"sources":["IndexSettingsManager.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-opensearch\";\nimport { IndexSettingsGetError, IndexSettingsSetError } from \"~/errors/index.js\";\nimport type { IIndexSettingsValues } from \"~/types.js\";\n\nexport class IndexSettingsManager {\n private readonly elasticsearch: Client;\n\n public constructor(elasticsearch: Client) {\n this.elasticsearch = elasticsearch;\n }\n\n public async getSettings(index: string): Promise<IIndexSettingsValues> {\n try {\n const response = await this.elasticsearch.indices.getSettings({\n index\n });\n\n const setting = response.body[index]?.settings?.index;\n\n return {\n numberOfReplicas: parseInt(String(setting?.number_of_replicas ?? \"0\"), 10),\n refreshInterval: setting?.refresh_interval ?? \"1s\"\n };\n } catch (ex) {\n throw new IndexSettingsGetError(ex, index);\n }\n }\n\n public async setSettings(index: string, settings: IIndexSettingsValues): Promise<void> {\n try {\n await this.elasticsearch.indices.putSettings({\n index,\n body: {\n index: {\n number_of_replicas: settings.numberOfReplicas,\n refresh_interval: settings.refreshInterval\n }\n }\n });\n } catch (ex) {\n throw new IndexSettingsSetError(ex, index);\n }\n }\n}\n"],"mappings":"AACA,SAASA,qBAAqB,EAAEC,qBAAqB;AAGrD,OAAO,MAAMC,oBAAoB,CAAC;EAGvBC,WAAWA,CAACC,aAAqB,EAAE;IACtC,IAAI,CAACA,aAAa,GAAGA,aAAa;EACtC;EAEA,MAAaC,WAAWA,CAACC,KAAa,EAAiC;IACnE,IAAI;MACA,MAAMC,QAAQ,GAAG,MAAM,IAAI,CAACH,aAAa,CAACI,OAAO,CAACH,WAAW,CAAC;QAC1DC;MACJ,CAAC,CAAC;MAEF,MAAMG,OAAO,GAAGF,QAAQ,CAACG,IAAI,CAACJ,KAAK,CAAC,EAAEK,QAAQ,EAAEL,KAAK;MAErD,OAAO;QACHM,gBAAgB,EAAEC,QAAQ,CAACC,MAAM,CAACL,OAAO,EAAEM,kBAAkB,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;QAC1EC,eAAe,EAAEP,OAAO,EAAEQ,gBAAgB,IAAI;MAClD,CAAC;IACL,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT,MAAM,IAAIlB,qBAAqB,CAACkB,EAAE,EAAEZ,KAAK,CAAC;IAC9C;EACJ;EAEA,MAAaa,WAAWA,CAACb,KAAa,EAAEK,QAA8B,EAAiB;IACnF,IAAI;MACA,MAAM,IAAI,CAACP,aAAa,CAACI,OAAO,CAACY,WAAW,CAAC;QACzCd,KAAK;QACLI,IAAI,EAAE;UACFJ,KAAK,EAAE;YACHS,kBAAkB,EAAEJ,QAAQ,CAACC,gBAAgB;YAC7CK,gBAAgB,EAAEN,QAAQ,CAACK;UAC/B;QACJ;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,MAAM,IAAIjB,qBAAqB,CAACiB,EAAE,EAAEZ,KAAK,CAAC;IAC9C;EACJ;AACJ","ignoreList":[]}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getTotalCount } from "@webiny/api-opensearch/types.js";
|
|
1
2
|
import { shouldIgnoreEsResponseError } from "./shouldIgnoreEsResponseError.js";
|
|
2
3
|
import { inspect } from "node:util";
|
|
3
4
|
export class ElasticsearchFetcher {
|
|
@@ -56,7 +57,7 @@ export class ElasticsearchFetcher {
|
|
|
56
57
|
return {
|
|
57
58
|
done: true,
|
|
58
59
|
cursor: undefined,
|
|
59
|
-
totalCount: total
|
|
60
|
+
totalCount: getTotalCount(total),
|
|
60
61
|
items: []
|
|
61
62
|
};
|
|
62
63
|
}
|
|
@@ -80,7 +81,7 @@ export class ElasticsearchFetcher {
|
|
|
80
81
|
return collection;
|
|
81
82
|
}, []);
|
|
82
83
|
return {
|
|
83
|
-
totalCount: total
|
|
84
|
+
totalCount: getTotalCount(total),
|
|
84
85
|
cursor: nextCursor,
|
|
85
86
|
done: !nextCursor,
|
|
86
87
|
items
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["shouldIgnoreEsResponseError","inspect","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","process","env","DEBUG","console","error","depth","showHidden","done","totalCount","items","hits","total","length","undefined","
|
|
1
|
+
{"version":3,"names":["getTotalCount","shouldIgnoreEsResponseError","inspect","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","process","env","DEBUG","console","error","depth","showHidden","done","totalCount","items","hits","total","length","undefined","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-opensearch\";\nimport type {\n IElasticsearchFetcher,\n IElasticsearchFetcherFetchParams,\n IElasticsearchFetcherFetchResponse,\n IElasticsearchFetcherFetchResponseItem\n} from \"./abstractions/ElasticsearchFetcher.js\";\nimport {\n type OpenSearchSearchResponse,\n type PrimitiveValue,\n getTotalCount\n} from \"@webiny/api-opensearch/types.js\";\nimport { shouldIgnoreEsResponseError } from \"./shouldIgnoreEsResponseError.js\";\nimport { inspect } from \"node:util\";\n\nexport interface IElasticsearchFetcherParams {\n client: Client;\n}\n\nexport class ElasticsearchFetcher implements IElasticsearchFetcher {\n private readonly client: Client;\n\n public constructor(params: IElasticsearchFetcherParams) {\n this.client = params.client;\n }\n public async fetch({\n index,\n cursor,\n limit\n }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse> {\n let response: OpenSearchSearchResponse;\n try {\n response = await this.client.search({\n index,\n body: {\n query: {\n match_all: {}\n },\n sort: {\n \"id.keyword\": {\n order: \"asc\"\n }\n },\n size: limit + 1,\n track_total_hits: true,\n search_after: cursor,\n _source: false\n }\n });\n } catch (ex) {\n /**\n * If we ignore the error, we can continue with the next index.\n */\n if (shouldIgnoreEsResponseError(ex)) {\n if (process.env.DEBUG === \"true\") {\n console.error(\n inspect(ex, {\n depth: 5,\n showHidden: true\n })\n );\n }\n return {\n done: true,\n totalCount: 0,\n items: []\n };\n }\n console.error(\"Failed to fetch data from Elasticsearch.\", ex);\n throw ex;\n }\n\n const { hits, total } = response.body.hits;\n if (hits.length === 0) {\n return {\n done: true,\n cursor: undefined,\n totalCount: getTotalCount(total),\n items: []\n };\n }\n\n const hasMoreItems = hits.length > limit;\n let nextCursor: PrimitiveValue[] | undefined;\n if (hasMoreItems) {\n hits.pop();\n nextCursor = hits.at(-1)?.sort;\n }\n const items = hits.reduce<IElasticsearchFetcherFetchResponseItem[]>((collection, hit) => {\n const [PK, SK] = hit._id.split(\":\");\n if (!PK || !SK) {\n return collection;\n }\n collection.push({\n PK,\n SK,\n _id: hit._id,\n index: hit._index\n });\n\n return collection;\n }, []);\n\n return {\n totalCount: getTotalCount(total),\n cursor: nextCursor,\n done: !nextCursor,\n items\n };\n }\n}\n"],"mappings":"AAOA,SAGIA,aAAa,QACV,iCAAiC;AACxC,SAASC,2BAA2B;AACpC,SAASC,OAAO,QAAQ,WAAW;AAMnC,OAAO,MAAMC,oBAAoB,CAAkC;EAGxDC,WAAWA,CAACC,MAAmC,EAAE;IACpD,IAAI,CAACC,MAAM,GAAGD,MAAM,CAACC,MAAM;EAC/B;EACA,MAAaC,KAAKA,CAAC;IACfC,KAAK;IACLC,MAAM;IACNC;EAC8B,CAAC,EAA+C;IAC9E,IAAIC,QAAkC;IACtC,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACL,MAAM,CAACM,MAAM,CAAC;QAChCJ,KAAK;QACLK,IAAI,EAAE;UACFC,KAAK,EAAE;YACHC,SAAS,EAAE,CAAC;UAChB,CAAC;UACDC,IAAI,EAAE;YACF,YAAY,EAAE;cACVC,KAAK,EAAE;YACX;UACJ,CAAC;UACDC,IAAI,EAAER,KAAK,GAAG,CAAC;UACfS,gBAAgB,EAAE,IAAI;UACtBC,YAAY,EAAEX,MAAM;UACpBY,OAAO,EAAE;QACb;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAIrB,2BAA2B,CAACqB,EAAE,CAAC,EAAE;QACjC,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9BC,OAAO,CAACC,KAAK,CACTzB,OAAO,CAACoB,EAAE,EAAE;YACRM,KAAK,EAAE,CAAC;YACRC,UAAU,EAAE;UAChB,CAAC,CACL,CAAC;QACL;QACA,OAAO;UACHC,IAAI,EAAE,IAAI;UACVC,UAAU,EAAE,CAAC;UACbC,KAAK,EAAE;QACX,CAAC;MACL;MACAN,OAAO,CAACC,KAAK,CAAC,0CAA0C,EAAEL,EAAE,CAAC;MAC7D,MAAMA,EAAE;IACZ;IAEA,MAAM;MAAEW,IAAI;MAAEC;IAAM,CAAC,GAAGvB,QAAQ,CAACE,IAAI,CAACoB,IAAI;IAC1C,IAAIA,IAAI,CAACE,MAAM,KAAK,CAAC,EAAE;MACnB,OAAO;QACHL,IAAI,EAAE,IAAI;QACVrB,MAAM,EAAE2B,SAAS;QACjBL,UAAU,EAAE/B,aAAa,CAACkC,KAAK,CAAC;QAChCF,KAAK,EAAE;MACX,CAAC;IACL;IAEA,MAAMK,YAAY,GAAGJ,IAAI,CAACE,MAAM,GAAGzB,KAAK;IACxC,IAAI4B,UAAwC;IAC5C,IAAID,YAAY,EAAE;MACdJ,IAAI,CAACM,GAAG,CAAC,CAAC;MACVD,UAAU,GAAGL,IAAI,CAACO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAExB,IAAI;IAClC;IACA,MAAMgB,KAAK,GAAGC,IAAI,CAACQ,MAAM,CAA2C,CAACC,UAAU,EAAEC,GAAG,KAAK;MACrF,MAAM,CAACC,EAAE,EAAEC,EAAE,CAAC,GAAGF,GAAG,CAACG,GAAG,CAACC,KAAK,CAAC,GAAG,CAAC;MACnC,IAAI,CAACH,EAAE,IAAI,CAACC,EAAE,EAAE;QACZ,OAAOH,UAAU;MACrB;MACAA,UAAU,CAACM,IAAI,CAAC;QACZJ,EAAE;QACFC,EAAE;QACFC,GAAG,EAAEH,GAAG,CAACG,GAAG;QACZtC,KAAK,EAAEmC,GAAG,CAACM;MACf,CAAC,CAAC;MAEF,OAAOP,UAAU;IACrB,CAAC,EAAE,EAAE,CAAC;IAEN,OAAO;MACHX,UAAU,EAAE/B,aAAa,CAACkC,KAAK,CAAC;MAChCzB,MAAM,EAAE6B,UAAU;MAClBR,IAAI,EAAE,CAACQ,UAAU;MACjBN;IACJ,CAAC;EACL;AACJ","ignoreList":[]}
|