@webiny/api-elasticsearch-tasks 6.0.0 → 6.1.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/abstractions/OpensearchTenantIndexFactory.d.ts +2 -2
- package/abstractions/OpensearchTenantIndexFactory.js.map +1 -1
- package/abstractions/SynchronizationContext.d.ts +3 -1
- package/abstractions/SynchronizationContext.js.map +1 -1
- package/helpers/getClients.d.ts +1 -1
- package/helpers/getClients.js +8 -6
- package/helpers/getClients.js.map +1 -1
- package/index.js +2 -1
- package/index.js.map +1 -1
- package/package.json +23 -23
- package/settings/IndexManager.d.ts +1 -1
- package/settings/IndexManager.js.map +1 -1
- package/settings/IndexSettingsManager.d.ts +1 -1
- package/settings/IndexSettingsManager.js.map +1 -1
- package/tasks/Manager.d.ts +3 -3
- package/tasks/Manager.js +3 -3
- package/tasks/Manager.js.map +1 -1
- package/tasks/dataSynchronization/DataSynchronizationTask.d.ts +5 -11
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.d.ts +1 -1
- package/tasks/dataSynchronization/elasticsearch/ElasticsearchFetcher.js.map +1 -1
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.d.ts +1 -1
- package/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js.map +1 -1
- package/tasks/dataSynchronization/types.d.ts +1 -1
- package/tasks/dataSynchronization/types.js.map +1 -1
- package/types.d.ts +4 -4
- package/types.js.map +1 -1
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { OpenSearchIndexRequestBody } from "@webiny/api-opensearch/types.js";
|
|
2
2
|
import type { Tenant } from "@webiny/api-core/types/tenancy.js";
|
|
3
3
|
export interface IOpensearchIndexConfig {
|
|
4
4
|
index: string;
|
|
5
|
-
settings?: Partial<
|
|
5
|
+
settings?: Partial<OpenSearchIndexRequestBody>;
|
|
6
6
|
}
|
|
7
7
|
export interface IOpensearchTenantIndexFactory {
|
|
8
8
|
getIndexList(tenant: Pick<Tenant, "id">): Promise<IOpensearchIndexConfig[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["createAbstraction","OpensearchTenantIndexFactory"],"sources":["OpensearchTenantIndexFactory.ts"],"sourcesContent":["import { createAbstraction } from \"@webiny/feature/api\";\nimport type {
|
|
1
|
+
{"version":3,"names":["createAbstraction","OpensearchTenantIndexFactory"],"sources":["OpensearchTenantIndexFactory.ts"],"sourcesContent":["import { createAbstraction } from \"@webiny/feature/api\";\nimport type { OpenSearchIndexRequestBody } from \"@webiny/api-opensearch/types.js\";\nimport type { Tenant } from \"@webiny/api-core/types/tenancy.js\";\n\nexport interface IOpensearchIndexConfig {\n index: string;\n settings?: Partial<OpenSearchIndexRequestBody>;\n}\n\nexport interface IOpensearchTenantIndexFactory {\n getIndexList(tenant: Pick<Tenant, \"id\">): Promise<IOpensearchIndexConfig[]>;\n}\n\nexport const OpensearchTenantIndexFactory = createAbstraction<IOpensearchTenantIndexFactory>(\n \"OpensearchTenantIndexFactory\"\n);\n\nexport namespace OpensearchTenantIndexFactory {\n export type Interface = IOpensearchTenantIndexFactory;\n export type IndexConfig = IOpensearchIndexConfig;\n}\n"],"mappings":"AAAA,SAASA,iBAAiB,QAAQ,qBAAqB;AAavD,OAAO,MAAMC,4BAA4B,GAAGD,iBAAiB,CACzD,8BACJ,CAAC","ignoreList":[]}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { Context } from "../types.js";
|
|
2
|
-
export type ISynchronizationContext = Pick<Context, "
|
|
2
|
+
export type ISynchronizationContext = Pick<Context, "opensearch"> & {
|
|
3
|
+
elasticsearch: Context["opensearch"];
|
|
4
|
+
};
|
|
3
5
|
export declare const SynchronizationContext: import("@webiny/di").Abstraction<ISynchronizationContext>;
|
|
4
6
|
export declare namespace SynchronizationContext {
|
|
5
7
|
type Interface = ISynchronizationContext;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["createAbstraction","SynchronizationContext"],"sources":["SynchronizationContext.ts"],"sourcesContent":["import { createAbstraction } from \"@webiny/feature/api\";\nimport type { Context } from \"~/types.js\";\n\nexport type ISynchronizationContext = Pick<Context, \"elasticsearch\"
|
|
1
|
+
{"version":3,"names":["createAbstraction","SynchronizationContext"],"sources":["SynchronizationContext.ts"],"sourcesContent":["import { createAbstraction } from \"@webiny/feature/api\";\nimport type { Context } from \"~/types.js\";\n\nexport type ISynchronizationContext = Pick<Context, \"opensearch\"> & {\n elasticsearch: Context[\"opensearch\"];\n};\n\nexport const SynchronizationContext =\n createAbstraction<ISynchronizationContext>(\"SynchronizationContext\");\n\nexport namespace SynchronizationContext {\n export type Interface = ISynchronizationContext;\n}\n"],"mappings":"AAAA,SAASA,iBAAiB,QAAQ,qBAAqB;AAOvD,OAAO,MAAMC,sBAAsB,GAC/BD,iBAAiB,CAA0B,wBAAwB,CAAC","ignoreList":[]}
|
package/helpers/getClients.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Context, IElasticsearchTaskConfig } from "../types.js";
|
|
2
2
|
export declare function getClients(context: Context, params?: Partial<IElasticsearchTaskConfig>): {
|
|
3
|
-
elasticsearchClient: import("@elastic/elasticsearch").Client;
|
|
4
3
|
documentClient: import("@aws-sdk/lib-dynamodb/dist-types/DynamoDBDocument").DynamoDBDocument;
|
|
4
|
+
elasticsearchClient: import("@opensearch-project/opensearch/.").Client;
|
|
5
5
|
};
|
package/helpers/getClients.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import { createElasticsearchClient } from "@webiny/api-elasticsearch";
|
|
2
1
|
import { getDocumentClient } from "@webiny/aws-sdk/client-dynamodb/index.js";
|
|
3
2
|
export function getClients(context, params) {
|
|
4
|
-
const elasticsearchClient = params?.elasticsearchClient ?? context.elasticsearch ?? createElasticsearchClient({
|
|
5
|
-
endpoint: `https://${process.env.OPENSEARCH_ENDPOINT}`
|
|
6
|
-
});
|
|
7
3
|
const documentClient = params?.documentClient ?? getDocumentClient();
|
|
4
|
+
if (params?.elasticsearchClient) {
|
|
5
|
+
return {
|
|
6
|
+
documentClient,
|
|
7
|
+
elasticsearchClient: params.elasticsearchClient
|
|
8
|
+
};
|
|
9
|
+
}
|
|
8
10
|
return {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
+
documentClient,
|
|
12
|
+
elasticsearchClient: context.opensearch
|
|
11
13
|
};
|
|
12
14
|
}
|
|
13
15
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["getDocumentClient","getClients","context","params","documentClient","elasticsearchClient","opensearch"],"sources":["getClients.ts"],"sourcesContent":["import { getDocumentClient } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport type { Context, IElasticsearchTaskConfig } from \"~/types.js\";\n\nexport function getClients(context: Context, params?: Partial<IElasticsearchTaskConfig>) {\n const documentClient = params?.documentClient ?? getDocumentClient();\n if (params?.elasticsearchClient) {\n return {\n documentClient,\n elasticsearchClient: params.elasticsearchClient\n };\n }\n\n return {\n documentClient,\n elasticsearchClient: context.opensearch\n };\n}\n"],"mappings":"AAAA,SAASA,iBAAiB,QAAQ,0CAA0C;AAG5E,OAAO,SAASC,UAAUA,CAACC,OAAgB,EAAEC,MAA0C,EAAE;EACrF,MAAMC,cAAc,GAAGD,MAAM,EAAEC,cAAc,IAAIJ,iBAAiB,CAAC,CAAC;EACpE,IAAIG,MAAM,EAAEE,mBAAmB,EAAE;IAC7B,OAAO;MACHD,cAAc;MACdC,mBAAmB,EAAEF,MAAM,CAACE;IAChC,CAAC;EACL;EAEA,OAAO;IACHD,cAAc;IACdC,mBAAmB,EAAEH,OAAO,CAACI;EACjC,CAAC;AACL","ignoreList":[]}
|
package/index.js
CHANGED
|
@@ -8,7 +8,8 @@ export const createElasticsearchBackgroundTasks = params => {
|
|
|
8
8
|
// @ts-expect-error We are going to remove this DB client.
|
|
9
9
|
context.container.registerInstance(DbRegistry, context.db.registry);
|
|
10
10
|
context.container.registerInstance(SynchronizationContext, {
|
|
11
|
-
|
|
11
|
+
opensearch: context.opensearch,
|
|
12
|
+
elasticsearch: context.opensearch
|
|
12
13
|
});
|
|
13
14
|
}), createElasticsearchReindexingTask(params), createEnableIndexingTask(params), createIndexesTaskDefinition(params), createDataSynchronization(params)];
|
|
14
15
|
};
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["createContextPlugin","createDataSynchronization","createElasticsearchReindexingTask","createEnableIndexingTask","createIndexesTaskDefinition","DbRegistry","SynchronizationContext","createElasticsearchBackgroundTasks","params","context","container","registerInstance","db","registry","elasticsearch"],"sources":["index.ts"],"sourcesContent":["import { createContextPlugin } from \"@webiny/api\";\nimport {\n createDataSynchronization,\n createElasticsearchReindexingTask,\n createEnableIndexingTask,\n createIndexesTaskDefinition\n} from \"~/tasks/index.js\";\nimport type { PluginCollection } from \"@webiny/plugins/types.js\";\nimport type { Context, IElasticsearchTaskConfig } from \"~/types.js\";\nimport { DbRegistry } from \"~/abstractions/DbRegistry.js\";\nimport { SynchronizationContext } from \"~/abstractions/SynchronizationContext.js\";\n\nexport type CreateElasticsearchBackgroundTasksParams = Partial<IElasticsearchTaskConfig>;\n\nexport const createElasticsearchBackgroundTasks = (\n params?: CreateElasticsearchBackgroundTasksParams\n): PluginCollection => {\n return [\n createContextPlugin<Context>(context => {\n // Register DbRegistry abstraction with the actual context.db.registry implementation\n // @ts-expect-error We are going to remove this DB client.\n context.container.registerInstance(DbRegistry, context.db.registry);\n\n context.container.registerInstance(SynchronizationContext, {\n elasticsearch: context.
|
|
1
|
+
{"version":3,"names":["createContextPlugin","createDataSynchronization","createElasticsearchReindexingTask","createEnableIndexingTask","createIndexesTaskDefinition","DbRegistry","SynchronizationContext","createElasticsearchBackgroundTasks","params","context","container","registerInstance","db","registry","opensearch","elasticsearch"],"sources":["index.ts"],"sourcesContent":["import { createContextPlugin } from \"@webiny/api\";\nimport {\n createDataSynchronization,\n createElasticsearchReindexingTask,\n createEnableIndexingTask,\n createIndexesTaskDefinition\n} from \"~/tasks/index.js\";\nimport type { PluginCollection } from \"@webiny/plugins/types.js\";\nimport type { Context, IElasticsearchTaskConfig } from \"~/types.js\";\nimport { DbRegistry } from \"~/abstractions/DbRegistry.js\";\nimport { SynchronizationContext } from \"~/abstractions/SynchronizationContext.js\";\n\nexport type CreateElasticsearchBackgroundTasksParams = Partial<IElasticsearchTaskConfig>;\n\nexport const createElasticsearchBackgroundTasks = (\n params?: CreateElasticsearchBackgroundTasksParams\n): PluginCollection => {\n return [\n createContextPlugin<Context>(context => {\n // Register DbRegistry abstraction with the actual context.db.registry implementation\n // @ts-expect-error We are going to remove this DB client.\n context.container.registerInstance(DbRegistry, context.db.registry);\n\n context.container.registerInstance(SynchronizationContext, {\n opensearch: context.opensearch,\n elasticsearch: context.opensearch\n });\n }),\n createElasticsearchReindexingTask(params),\n createEnableIndexingTask(params),\n createIndexesTaskDefinition(params),\n createDataSynchronization(params)\n ];\n};\n\nexport * from \"./abstractions/OpensearchTenantIndexFactory.js\";\n"],"mappings":"AAAA,SAASA,mBAAmB,QAAQ,aAAa;AACjD,SACIC,yBAAyB,EACzBC,iCAAiC,EACjCC,wBAAwB,EACxBC,2BAA2B;AAI/B,SAASC,UAAU;AACnB,SAASC,sBAAsB;AAI/B,OAAO,MAAMC,kCAAkC,GAC3CC,MAAiD,IAC9B;EACnB,OAAO,CACHR,mBAAmB,CAAUS,OAAO,IAAI;IACpC;IACA;IACAA,OAAO,CAACC,SAAS,CAACC,gBAAgB,CAACN,UAAU,EAAEI,OAAO,CAACG,EAAE,CAACC,QAAQ,CAAC;IAEnEJ,OAAO,CAACC,SAAS,CAACC,gBAAgB,CAACL,sBAAsB,EAAE;MACvDQ,UAAU,EAAEL,OAAO,CAACK,UAAU;MAC9BC,aAAa,EAAEN,OAAO,CAACK;IAC3B,CAAC,CAAC;EACN,CAAC,CAAC,EACFZ,iCAAiC,CAACM,MAAM,CAAC,EACzCL,wBAAwB,CAACK,MAAM,CAAC,EAChCJ,2BAA2B,CAACI,MAAM,CAAC,EACnCP,yBAAyB,CAACO,MAAM,CAAC,CACpC;AACL,CAAC;AAED","ignoreList":[]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webiny/api-elasticsearch-tasks",
|
|
3
|
-
"version": "6.0.0",
|
|
3
|
+
"version": "6.1.0-beta.0",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"repository": {
|
|
@@ -13,38 +13,38 @@
|
|
|
13
13
|
],
|
|
14
14
|
"license": "MIT",
|
|
15
15
|
"dependencies": {
|
|
16
|
-
"@webiny/api": "6.0.0",
|
|
17
|
-
"@webiny/api-core": "6.0.0",
|
|
18
|
-
"@webiny/api-dynamodb-to-elasticsearch": "6.0.0",
|
|
19
|
-
"@webiny/api-
|
|
20
|
-
"@webiny/aws-sdk": "6.0.0",
|
|
21
|
-
"@webiny/db": "6.0.0",
|
|
22
|
-
"@webiny/db-dynamodb": "6.0.0",
|
|
23
|
-
"@webiny/error": "6.0.0",
|
|
24
|
-
"@webiny/feature": "6.0.0",
|
|
25
|
-
"@webiny/plugins": "6.0.0",
|
|
26
|
-
"@webiny/tasks": "6.0.0",
|
|
27
|
-
"@webiny/utils": "6.0.0"
|
|
16
|
+
"@webiny/api": "6.1.0-beta.0",
|
|
17
|
+
"@webiny/api-core": "6.1.0-beta.0",
|
|
18
|
+
"@webiny/api-dynamodb-to-elasticsearch": "6.1.0-beta.0",
|
|
19
|
+
"@webiny/api-opensearch": "6.1.0-beta.0",
|
|
20
|
+
"@webiny/aws-sdk": "6.1.0-beta.0",
|
|
21
|
+
"@webiny/db": "6.1.0-beta.0",
|
|
22
|
+
"@webiny/db-dynamodb": "6.1.0-beta.0",
|
|
23
|
+
"@webiny/error": "6.1.0-beta.0",
|
|
24
|
+
"@webiny/feature": "6.1.0-beta.0",
|
|
25
|
+
"@webiny/plugins": "6.1.0-beta.0",
|
|
26
|
+
"@webiny/tasks": "6.1.0-beta.0",
|
|
27
|
+
"@webiny/utils": "6.1.0-beta.0"
|
|
28
28
|
},
|
|
29
29
|
"devDependencies": {
|
|
30
30
|
"@webiny/api": "0.0.0",
|
|
31
|
-
"@webiny/api-headless-cms": "6.0.0",
|
|
32
|
-
"@webiny/build-tools": "6.0.0",
|
|
33
|
-
"@webiny/handler": "6.0.0",
|
|
34
|
-
"@webiny/handler-aws": "6.0.0",
|
|
35
|
-
"@webiny/handler-db": "6.0.0",
|
|
36
|
-
"@webiny/handler-graphql": "6.0.0",
|
|
37
|
-
"@webiny/project-utils": "6.0.0",
|
|
31
|
+
"@webiny/api-headless-cms": "6.1.0-beta.0",
|
|
32
|
+
"@webiny/build-tools": "6.1.0-beta.0",
|
|
33
|
+
"@webiny/handler": "6.1.0-beta.0",
|
|
34
|
+
"@webiny/handler-aws": "6.1.0-beta.0",
|
|
35
|
+
"@webiny/handler-db": "6.1.0-beta.0",
|
|
36
|
+
"@webiny/handler-graphql": "6.1.0-beta.0",
|
|
37
|
+
"@webiny/project-utils": "6.1.0-beta.0",
|
|
38
38
|
"rimraf": "6.1.3",
|
|
39
|
-
"type-fest": "5.
|
|
39
|
+
"type-fest": "5.5.0",
|
|
40
40
|
"typescript": "5.9.3",
|
|
41
|
-
"vitest": "4.
|
|
41
|
+
"vitest": "4.1.2"
|
|
42
42
|
},
|
|
43
43
|
"publishConfig": {
|
|
44
44
|
"access": "public",
|
|
45
45
|
"directory": "dist"
|
|
46
46
|
},
|
|
47
|
-
"gitHead": "
|
|
47
|
+
"gitHead": "a3bd3695c66c79238e380d7360d9731b5fcf9c87",
|
|
48
48
|
"adio": {
|
|
49
49
|
"ignore": {
|
|
50
50
|
"src": [
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { IElasticsearchIndexingTaskValuesSettings, IIndexSettingsValues } from "../types.js";
|
|
2
2
|
import type { IIndexManager } from "../settings/types.js";
|
|
3
|
-
import type { Client } from "@webiny/api-
|
|
3
|
+
import type { Client } from "@webiny/api-opensearch";
|
|
4
4
|
export interface IListIndicesResponse {
|
|
5
5
|
index: string;
|
|
6
6
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["IndexSettingsManager","DisableIndexing","EnableIndexing","getObjectProperties","defaultIndexSettings","numberOfReplicas","refreshInterval","indexPrefix","process","env","OPENSEARCH_INDEX_PREFIX","filterIndex","item","startsWith","IndexManager","settings","_settings","constructor","client","defaults","indexSettings","disable","enable","list","response","cat","indices","format","Array","isArray","body","map","index","filter","ex","console","error","JSON","stringify","message","disableIndexing","exec","enableIndexing","indexes","Object","keys","createIndex","create","indexExists","exists","ignore_unavailable","allow_no_indices","include_defaults","flat_settings","local"],"sources":["IndexManager.ts"],"sourcesContent":["import { IndexSettingsManager } from \"~/settings/IndexSettingsManager.js\";\nimport { DisableIndexing } from \"./DisableIndexing.js\";\nimport { EnableIndexing } from \"./EnableIndexing.js\";\nimport type { IElasticsearchIndexingTaskValuesSettings, IIndexSettingsValues } from \"~/types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { Client } from \"@webiny/api-
|
|
1
|
+
{"version":3,"names":["IndexSettingsManager","DisableIndexing","EnableIndexing","getObjectProperties","defaultIndexSettings","numberOfReplicas","refreshInterval","indexPrefix","process","env","OPENSEARCH_INDEX_PREFIX","filterIndex","item","startsWith","IndexManager","settings","_settings","constructor","client","defaults","indexSettings","disable","enable","list","response","cat","indices","format","Array","isArray","body","map","index","filter","ex","console","error","JSON","stringify","message","disableIndexing","exec","enableIndexing","indexes","Object","keys","createIndex","create","indexExists","exists","ignore_unavailable","allow_no_indices","include_defaults","flat_settings","local"],"sources":["IndexManager.ts"],"sourcesContent":["import { IndexSettingsManager } from \"~/settings/IndexSettingsManager.js\";\nimport { DisableIndexing } from \"./DisableIndexing.js\";\nimport { EnableIndexing } from \"./EnableIndexing.js\";\nimport type { IElasticsearchIndexingTaskValuesSettings, IIndexSettingsValues } from \"~/types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { Client } from \"@webiny/api-opensearch\";\nimport { getObjectProperties } from \"@webiny/utils\";\n\nconst defaultIndexSettings: IIndexSettingsValues = {\n numberOfReplicas: 1,\n refreshInterval: \"1s\"\n};\n\nexport interface IListIndicesResponse {\n index: string;\n}\n\nconst indexPrefix = process.env.OPENSEARCH_INDEX_PREFIX || \"\";\nconst filterIndex = (item?: string) => {\n if (!item) {\n return false;\n } else if (item.startsWith(\".\")) {\n return false;\n } else if (indexPrefix) {\n return item.startsWith(indexPrefix);\n }\n return true;\n};\n\nexport class IndexManager implements IIndexManager {\n private readonly client: Client;\n private readonly disable: DisableIndexing;\n private readonly enable: EnableIndexing;\n private readonly _settings: IElasticsearchIndexingTaskValuesSettings;\n\n private readonly defaults: IIndexSettingsValues;\n\n public get settings(): IElasticsearchIndexingTaskValuesSettings {\n return this._settings;\n }\n\n public constructor(\n client: Client,\n settings: IElasticsearchIndexingTaskValuesSettings,\n defaults?: Partial<IIndexSettingsValues>\n ) {\n const indexSettings = new IndexSettingsManager(client);\n this.client = client;\n this.disable = new DisableIndexing(indexSettings);\n this.enable = new EnableIndexing(indexSettings);\n this._settings = settings;\n this.defaults = {\n refreshInterval: defaults?.refreshInterval || defaultIndexSettings.refreshInterval,\n numberOfReplicas: defaults?.numberOfReplicas || defaultIndexSettings.numberOfReplicas\n };\n }\n\n public async list(): Promise<string[]> {\n try {\n const response = await this.client.cat.indices<IListIndicesResponse[]>({\n format: \"json\"\n });\n if (!Array.isArray(response.body)) {\n return [];\n }\n return response.body.map(item => item.index).filter(filterIndex);\n } catch (ex) {\n console.error(\n JSON.stringify({\n message: \"Failed to list indices.\",\n error: getObjectProperties(ex)\n })\n );\n return [];\n }\n }\n\n public async disableIndexing(index: string) {\n /**\n * No need to disable indexing if it's already disabled.\n */\n if (this._settings[index]) {\n return this._settings[index];\n }\n const settings = await this.disable.exec(index);\n this._settings[index] = settings;\n return settings;\n }\n\n public async enableIndexing(index?: string) {\n if (!index) {\n const indexes = Object.keys(this._settings);\n for (const index of indexes) {\n await this.enableIndexing(index);\n }\n return;\n }\n const settings = this._settings[index] || this.defaults;\n await this.enable.exec(index, settings);\n }\n\n public async createIndex(index: string, settings?: Record<string, any>): Promise<void> {\n await this.client.indices.create({\n index,\n body: settings\n });\n }\n\n public async indexExists(index: string): Promise<boolean> {\n const response = await this.client.indices.exists({\n index,\n ignore_unavailable: false,\n allow_no_indices: true,\n include_defaults: true,\n flat_settings: false,\n local: false\n });\n return !!response.body;\n }\n}\n"],"mappings":"AAAA,SAASA,oBAAoB;AAC7B,SAASC,eAAe;AACxB,SAASC,cAAc;AAIvB,SAASC,mBAAmB,QAAQ,eAAe;AAEnD,MAAMC,oBAA0C,GAAG;EAC/CC,gBAAgB,EAAE,CAAC;EACnBC,eAAe,EAAE;AACrB,CAAC;AAMD,MAAMC,WAAW,GAAGC,OAAO,CAACC,GAAG,CAACC,uBAAuB,IAAI,EAAE;AAC7D,MAAMC,WAAW,GAAIC,IAAa,IAAK;EACnC,IAAI,CAACA,IAAI,EAAE;IACP,OAAO,KAAK;EAChB,CAAC,MAAM,IAAIA,IAAI,CAACC,UAAU,CAAC,GAAG,CAAC,EAAE;IAC7B,OAAO,KAAK;EAChB,CAAC,MAAM,IAAIN,WAAW,EAAE;IACpB,OAAOK,IAAI,CAACC,UAAU,CAACN,WAAW,CAAC;EACvC;EACA,OAAO,IAAI;AACf,CAAC;AAED,OAAO,MAAMO,YAAY,CAA0B;EAQ/C,IAAWC,QAAQA,CAAA,EAA6C;IAC5D,OAAO,IAAI,CAACC,SAAS;EACzB;EAEOC,WAAWA,CACdC,MAAc,EACdH,QAAkD,EAClDI,QAAwC,EAC1C;IACE,MAAMC,aAAa,GAAG,IAAIpB,oBAAoB,CAACkB,MAAM,CAAC;IACtD,IAAI,CAACA,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACG,OAAO,GAAG,IAAIpB,eAAe,CAACmB,aAAa,CAAC;IACjD,IAAI,CAACE,MAAM,GAAG,IAAIpB,cAAc,CAACkB,aAAa,CAAC;IAC/C,IAAI,CAACJ,SAAS,GAAGD,QAAQ;IACzB,IAAI,CAACI,QAAQ,GAAG;MACZb,eAAe,EAAEa,QAAQ,EAAEb,eAAe,IAAIF,oBAAoB,CAACE,eAAe;MAClFD,gBAAgB,EAAEc,QAAQ,EAAEd,gBAAgB,IAAID,oBAAoB,CAACC;IACzE,CAAC;EACL;EAEA,MAAakB,IAAIA,CAAA,EAAsB;IACnC,IAAI;MACA,MAAMC,QAAQ,GAAG,MAAM,IAAI,CAACN,MAAM,CAACO,GAAG,CAACC,OAAO,CAAyB;QACnEC,MAAM,EAAE;MACZ,CAAC,CAAC;MACF,IAAI,CAACC,KAAK,CAACC,OAAO,CAACL,QAAQ,CAACM,IAAI,CAAC,EAAE;QAC/B,OAAO,EAAE;MACb;MACA,OAAON,QAAQ,CAACM,IAAI,CAACC,GAAG,CAACnB,IAAI,IAAIA,IAAI,CAACoB,KAAK,CAAC,CAACC,MAAM,CAACtB,WAAW,CAAC;IACpE,CAAC,CAAC,OAAOuB,EAAE,EAAE;MACTC,OAAO,CAACC,KAAK,CACTC,IAAI,CAACC,SAAS,CAAC;QACXC,OAAO,EAAE,yBAAyB;QAClCH,KAAK,EAAEjC,mBAAmB,CAAC+B,EAAE;MACjC,CAAC,CACL,CAAC;MACD,OAAO,EAAE;IACb;EACJ;EAEA,MAAaM,eAAeA,CAACR,KAAa,EAAE;IACxC;AACR;AACA;IACQ,IAAI,IAAI,CAAChB,SAAS,CAACgB,KAAK,CAAC,EAAE;MACvB,OAAO,IAAI,CAAChB,SAAS,CAACgB,KAAK,CAAC;IAChC;IACA,MAAMjB,QAAQ,GAAG,MAAM,IAAI,CAACM,OAAO,CAACoB,IAAI,CAACT,KAAK,CAAC;IAC/C,IAAI,CAAChB,SAAS,CAACgB,KAAK,CAAC,GAAGjB,QAAQ;IAChC,OAAOA,QAAQ;EACnB;EAEA,MAAa2B,cAAcA,CAACV,KAAc,EAAE;IACxC,IAAI,CAACA,KAAK,EAAE;MACR,MAAMW,OAAO,GAAGC,MAAM,CAACC,IAAI,CAAC,IAAI,CAAC7B,SAAS,CAAC;MAC3C,KAAK,MAAMgB,KAAK,IAAIW,OAAO,EAAE;QACzB,MAAM,IAAI,CAACD,cAAc,CAACV,KAAK,CAAC;MACpC;MACA;IACJ;IACA,MAAMjB,QAAQ,GAAG,IAAI,CAACC,SAAS,CAACgB,KAAK,CAAC,IAAI,IAAI,CAACb,QAAQ;IACvD,MAAM,IAAI,CAACG,MAAM,CAACmB,IAAI,CAACT,KAAK,EAAEjB,QAAQ,CAAC;EAC3C;EAEA,MAAa+B,WAAWA,CAACd,KAAa,EAAEjB,QAA8B,EAAiB;IACnF,MAAM,IAAI,CAACG,MAAM,CAACQ,OAAO,CAACqB,MAAM,CAAC;MAC7Bf,KAAK;MACLF,IAAI,EAAEf;IACV,CAAC,CAAC;EACN;EAEA,MAAaiC,WAAWA,CAAChB,KAAa,EAAoB;IACtD,MAAMR,QAAQ,GAAG,MAAM,IAAI,CAACN,MAAM,CAACQ,OAAO,CAACuB,MAAM,CAAC;MAC9CjB,KAAK;MACLkB,kBAAkB,EAAE,KAAK;MACzBC,gBAAgB,EAAE,IAAI;MACtBC,gBAAgB,EAAE,IAAI;MACtBC,aAAa,EAAE,KAAK;MACpBC,KAAK,EAAE;IACX,CAAC,CAAC;IACF,OAAO,CAAC,CAAC9B,QAAQ,CAACM,IAAI;EAC1B;AACJ","ignoreList":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["IndexSettingsGetError","IndexSettingsSetError","IndexSettingsManager","constructor","elasticsearch","getSettings","index","response","indices","setting","body","settings","numberOfReplicas","number_of_replicas","refreshInterval","refresh_interval","ex","setSettings","putSettings"],"sources":["IndexSettingsManager.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-
|
|
1
|
+
{"version":3,"names":["IndexSettingsGetError","IndexSettingsSetError","IndexSettingsManager","constructor","elasticsearch","getSettings","index","response","indices","setting","body","settings","numberOfReplicas","number_of_replicas","refreshInterval","refresh_interval","ex","setSettings","putSettings"],"sources":["IndexSettingsManager.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-opensearch\";\nimport { IndexSettingsGetError, IndexSettingsSetError } from \"~/errors/index.js\";\nimport type { IIndexSettingsValues } from \"~/types.js\";\n\nexport class IndexSettingsManager {\n private readonly elasticsearch: Client;\n\n public constructor(elasticsearch: Client) {\n this.elasticsearch = elasticsearch;\n }\n\n public async getSettings(index: string): Promise<IIndexSettingsValues> {\n try {\n const response = await this.elasticsearch.indices.getSettings({\n index\n });\n\n const setting = response.body[index].settings.index;\n\n return {\n numberOfReplicas: setting.number_of_replicas,\n refreshInterval: setting.refresh_interval\n };\n } catch (ex) {\n throw new IndexSettingsGetError(ex, index);\n }\n }\n\n public async setSettings(index: string, settings: IIndexSettingsValues): Promise<void> {\n try {\n await this.elasticsearch.indices.putSettings({\n index,\n body: {\n index: {\n number_of_replicas: settings.numberOfReplicas,\n refresh_interval: settings.refreshInterval\n }\n }\n });\n } catch (ex) {\n throw new IndexSettingsSetError(ex, index);\n }\n }\n}\n"],"mappings":"AACA,SAASA,qBAAqB,EAAEC,qBAAqB;AAGrD,OAAO,MAAMC,oBAAoB,CAAC;EAGvBC,WAAWA,CAACC,aAAqB,EAAE;IACtC,IAAI,CAACA,aAAa,GAAGA,aAAa;EACtC;EAEA,MAAaC,WAAWA,CAACC,KAAa,EAAiC;IACnE,IAAI;MACA,MAAMC,QAAQ,GAAG,MAAM,IAAI,CAACH,aAAa,CAACI,OAAO,CAACH,WAAW,CAAC;QAC1DC;MACJ,CAAC,CAAC;MAEF,MAAMG,OAAO,GAAGF,QAAQ,CAACG,IAAI,CAACJ,KAAK,CAAC,CAACK,QAAQ,CAACL,KAAK;MAEnD,OAAO;QACHM,gBAAgB,EAAEH,OAAO,CAACI,kBAAkB;QAC5CC,eAAe,EAAEL,OAAO,CAACM;MAC7B,CAAC;IACL,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT,MAAM,IAAIhB,qBAAqB,CAACgB,EAAE,EAAEV,KAAK,CAAC;IAC9C;EACJ;EAEA,MAAaW,WAAWA,CAACX,KAAa,EAAEK,QAA8B,EAAiB;IACnF,IAAI;MACA,MAAM,IAAI,CAACP,aAAa,CAACI,OAAO,CAACU,WAAW,CAAC;QACzCZ,KAAK;QACLI,IAAI,EAAE;UACFJ,KAAK,EAAE;YACHO,kBAAkB,EAAEF,QAAQ,CAACC,gBAAgB;YAC7CG,gBAAgB,EAAEJ,QAAQ,CAACG;UAC/B;QACJ;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOE,EAAE,EAAE;MACT,MAAM,IAAIf,qBAAqB,CAACe,EAAE,EAAEV,KAAK,CAAC;IAC9C;EACJ;AACJ","ignoreList":[]}
|
package/tasks/Manager.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { TaskController } from "@webiny/api-core/features/task/TaskController/index.js";
|
|
2
2
|
import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb/index.js";
|
|
3
|
-
import type { Client } from "@webiny/api-
|
|
4
|
-
import {
|
|
3
|
+
import type { Client } from "@webiny/api-opensearch";
|
|
4
|
+
import { createOpenSearchTable } from "@webiny/api-opensearch";
|
|
5
5
|
import type { IManager } from "../types.js";
|
|
6
6
|
import type { BatchReadItem } from "@webiny/db-dynamodb/utils/batch/batchRead.js";
|
|
7
7
|
import type { IEntity } from "@webiny/db-dynamodb";
|
|
@@ -15,7 +15,7 @@ export declare class Manager<T extends TaskDefinition.TaskInput, O extends TaskD
|
|
|
15
15
|
readonly controller: TaskController.Interface<T, O>;
|
|
16
16
|
readonly documentClient: DynamoDBDocument;
|
|
17
17
|
readonly elasticsearch: Client;
|
|
18
|
-
readonly table: ReturnType<typeof
|
|
18
|
+
readonly table: ReturnType<typeof createOpenSearchTable>;
|
|
19
19
|
private readonly entities;
|
|
20
20
|
constructor(params: ManagerParams<T, O>);
|
|
21
21
|
getEntity(name: string): IEntity;
|
package/tasks/Manager.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createOpenSearchEntity, createOpenSearchTable } from "@webiny/api-opensearch";
|
|
2
2
|
import { batchReadAll } from "@webiny/db-dynamodb/utils/batch/batchRead.js";
|
|
3
3
|
export class Manager {
|
|
4
4
|
entities = {};
|
|
@@ -6,7 +6,7 @@ export class Manager {
|
|
|
6
6
|
this.controller = params.controller;
|
|
7
7
|
this.documentClient = params.documentClient;
|
|
8
8
|
this.elasticsearch = params.elasticsearchClient;
|
|
9
|
-
this.table =
|
|
9
|
+
this.table = createOpenSearchTable({
|
|
10
10
|
documentClient: this.documentClient
|
|
11
11
|
});
|
|
12
12
|
}
|
|
@@ -14,7 +14,7 @@ export class Manager {
|
|
|
14
14
|
if (this.entities[name]) {
|
|
15
15
|
return this.entities[name];
|
|
16
16
|
}
|
|
17
|
-
return this.entities[name] =
|
|
17
|
+
return this.entities[name] = createOpenSearchEntity({
|
|
18
18
|
table: this.table,
|
|
19
19
|
entityName: name
|
|
20
20
|
});
|
package/tasks/Manager.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["
|
|
1
|
+
{"version":3,"names":["createOpenSearchEntity","createOpenSearchTable","batchReadAll","Manager","entities","constructor","params","controller","documentClient","elasticsearch","elasticsearchClient","table","getEntity","name","entityName","read","items"],"sources":["Manager.ts"],"sourcesContent":["import { TaskController } from \"@webiny/api-core/features/task/TaskController/index.js\";\nimport type { DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport type { Client } from \"@webiny/api-opensearch\";\nimport { createOpenSearchEntity, createOpenSearchTable } from \"@webiny/api-opensearch\";\nimport type { IManager } from \"~/types.js\";\nimport type { BatchReadItem } from \"@webiny/db-dynamodb/utils/batch/batchRead.js\";\nimport { batchReadAll } from \"@webiny/db-dynamodb/utils/batch/batchRead.js\";\nimport type { IEntity } from \"@webiny/db-dynamodb\";\nimport { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\n\nexport interface ManagerParams<\n T extends TaskDefinition.TaskInput,\n O extends TaskDefinition.TaskOutput\n> {\n documentClient: DynamoDBDocument;\n elasticsearchClient: Client;\n controller: TaskController.Interface<T, O>;\n}\n\nexport class Manager<\n T extends TaskDefinition.TaskInput,\n O extends TaskDefinition.TaskOutput = TaskDefinition.TaskOutput\n> implements IManager<T, O>\n{\n public readonly controller: TaskController.Interface<T, O>;\n public readonly documentClient: DynamoDBDocument;\n public readonly elasticsearch: Client;\n public readonly table: ReturnType<typeof createOpenSearchTable>;\n\n private readonly entities: Record<string, IEntity> = {};\n\n public constructor(params: ManagerParams<T, O>) {\n this.controller = params.controller;\n this.documentClient = params.documentClient;\n this.elasticsearch = params.elasticsearchClient;\n\n this.table = createOpenSearchTable({\n documentClient: this.documentClient\n });\n }\n\n public getEntity(name: string): IEntity {\n if (this.entities[name]) {\n return this.entities[name];\n }\n\n return (this.entities[name] = createOpenSearchEntity({\n table: this.table,\n entityName: name\n }));\n }\n\n public async read<T>(items: BatchReadItem[]): Promise<T[]> {\n return await batchReadAll<T>({\n table: this.table.table,\n items\n });\n }\n}\n"],"mappings":"AAGA,SAASA,sBAAsB,EAAEC,qBAAqB,QAAQ,wBAAwB;AAGtF,SAASC,YAAY,QAAQ,8CAA8C;AAa3E,OAAO,MAAMC,OAAO,CAIpB;EAMqBC,QAAQ,GAA4B,CAAC,CAAC;EAEhDC,WAAWA,CAACC,MAA2B,EAAE;IAC5C,IAAI,CAACC,UAAU,GAAGD,MAAM,CAACC,UAAU;IACnC,IAAI,CAACC,cAAc,GAAGF,MAAM,CAACE,cAAc;IAC3C,IAAI,CAACC,aAAa,GAAGH,MAAM,CAACI,mBAAmB;IAE/C,IAAI,CAACC,KAAK,GAAGV,qBAAqB,CAAC;MAC/BO,cAAc,EAAE,IAAI,CAACA;IACzB,CAAC,CAAC;EACN;EAEOI,SAASA,CAACC,IAAY,EAAW;IACpC,IAAI,IAAI,CAACT,QAAQ,CAACS,IAAI,CAAC,EAAE;MACrB,OAAO,IAAI,CAACT,QAAQ,CAACS,IAAI,CAAC;IAC9B;IAEA,OAAQ,IAAI,CAACT,QAAQ,CAACS,IAAI,CAAC,GAAGb,sBAAsB,CAAC;MACjDW,KAAK,EAAE,IAAI,CAACA,KAAK;MACjBG,UAAU,EAAED;IAChB,CAAC,CAAC;EACN;EAEA,MAAaE,IAAIA,CAAIC,KAAsB,EAAgB;IACvD,OAAO,MAAMd,YAAY,CAAI;MACzBS,KAAK,EAAE,IAAI,CAACA,KAAK,CAACA,KAAK;MACvBK;IACJ,CAAC,CAAC;EACN;AACJ","ignoreList":[]}
|
|
@@ -16,19 +16,13 @@ export declare class DataSynchronizationTask implements TaskDefinition.Interface
|
|
|
16
16
|
constructor(elasticsearchClient: IElasticsearchTaskConfig["elasticsearchClient"], documentClient: IElasticsearchTaskConfig["documentClient"], elasticsearchSynchronize: ElasticsearchSynchronize);
|
|
17
17
|
run({ input, controller }: TaskDefinition.RunParams<IDataSynchronizationInput, IDataSynchronizationOutput>): Promise<import("~/tasks/dataSynchronization/types.js").ISynchronizationRunResult>;
|
|
18
18
|
createInputValidation({ validator }: TaskDefinition.CreateInputValidationParams): {
|
|
19
|
-
flow: import("zod").ZodEnum<
|
|
19
|
+
flow: import("zod").ZodEnum<{
|
|
20
|
+
elasticsearchToDynamoDb: "elasticsearchToDynamoDb";
|
|
21
|
+
}>;
|
|
20
22
|
elasticsearchToDynamoDb: import("zod").ZodDefault<import("zod").ZodOptional<import("zod").ZodObject<{
|
|
21
23
|
finished: import("zod").ZodDefault<import("zod").ZodOptional<import("zod").ZodBoolean>>;
|
|
22
24
|
index: import("zod").ZodOptional<import("zod").ZodString>;
|
|
23
|
-
cursor: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString
|
|
24
|
-
},
|
|
25
|
-
finished: boolean;
|
|
26
|
-
index?: string | undefined;
|
|
27
|
-
cursor?: string[] | undefined;
|
|
28
|
-
}, {
|
|
29
|
-
index?: string | undefined;
|
|
30
|
-
cursor?: string[] | undefined;
|
|
31
|
-
finished?: boolean | undefined;
|
|
32
|
-
}>>>;
|
|
25
|
+
cursor: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString>>;
|
|
26
|
+
}, import("zod/v4/core").$strip>>>;
|
|
33
27
|
};
|
|
34
28
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Client } from "@webiny/api-
|
|
1
|
+
import type { Client } from "@webiny/api-opensearch";
|
|
2
2
|
import type { IElasticsearchFetcher, IElasticsearchFetcherFetchParams, IElasticsearchFetcherFetchResponse } from "./abstractions/ElasticsearchFetcher.js";
|
|
3
3
|
export interface IElasticsearchFetcherParams {
|
|
4
4
|
client: Client;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":["shouldIgnoreEsResponseError","inspect","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","process","env","DEBUG","console","error","depth","showHidden","done","totalCount","items","hits","total","length","undefined","value","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-
|
|
1
|
+
{"version":3,"names":["shouldIgnoreEsResponseError","inspect","ElasticsearchFetcher","constructor","params","client","fetch","index","cursor","limit","response","search","body","query","match_all","sort","order","size","track_total_hits","search_after","_source","ex","process","env","DEBUG","console","error","depth","showHidden","done","totalCount","items","hits","total","length","undefined","value","hasMoreItems","nextCursor","pop","at","reduce","collection","hit","PK","SK","_id","split","push","_index"],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { Client } from \"@webiny/api-opensearch\";\nimport type {\n IElasticsearchFetcher,\n IElasticsearchFetcherFetchParams,\n IElasticsearchFetcherFetchResponse,\n IElasticsearchFetcherFetchResponseItem\n} from \"./abstractions/ElasticsearchFetcher.js\";\nimport type { OpenSearchSearchResponse, PrimitiveValue } from \"@webiny/api-opensearch/types.js\";\nimport { shouldIgnoreEsResponseError } from \"./shouldIgnoreEsResponseError.js\";\nimport { inspect } from \"node:util\";\n\nexport interface IElasticsearchFetcherParams {\n client: Client;\n}\n\nexport class ElasticsearchFetcher implements IElasticsearchFetcher {\n private readonly client: Client;\n\n public constructor(params: IElasticsearchFetcherParams) {\n this.client = params.client;\n }\n public async fetch({\n index,\n cursor,\n limit\n }: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse> {\n let response: OpenSearchSearchResponse<undefined>;\n try {\n response = await this.client.search({\n index,\n body: {\n query: {\n match_all: {}\n },\n sort: {\n \"id.keyword\": {\n order: \"asc\"\n }\n },\n size: limit + 1,\n track_total_hits: true,\n search_after: cursor,\n _source: false\n }\n });\n } catch (ex) {\n /**\n * If we ignore the error, we can continue with the next index.\n */\n if (shouldIgnoreEsResponseError(ex)) {\n if (process.env.DEBUG === \"true\") {\n console.error(\n inspect(ex, {\n depth: 5,\n showHidden: true\n })\n );\n }\n return {\n done: true,\n totalCount: 0,\n items: []\n };\n }\n console.error(\"Failed to fetch data from Elasticsearch.\", ex);\n throw ex;\n }\n\n const { hits, total } = response.body.hits;\n if (hits.length === 0) {\n return {\n done: true,\n cursor: undefined,\n totalCount: total.value,\n items: []\n };\n }\n\n const hasMoreItems = hits.length > limit;\n let nextCursor: PrimitiveValue[] | undefined;\n if (hasMoreItems) {\n hits.pop();\n nextCursor = hits.at(-1)?.sort;\n }\n const items = hits.reduce<IElasticsearchFetcherFetchResponseItem[]>((collection, hit) => {\n const [PK, SK] = hit._id.split(\":\");\n if (!PK || !SK) {\n return collection;\n }\n collection.push({\n PK,\n SK,\n _id: hit._id,\n index: hit._index\n });\n\n return collection;\n }, []);\n\n return {\n totalCount: total.value,\n cursor: nextCursor,\n done: !nextCursor,\n items\n };\n }\n}\n"],"mappings":"AAQA,SAASA,2BAA2B;AACpC,SAASC,OAAO,QAAQ,WAAW;AAMnC,OAAO,MAAMC,oBAAoB,CAAkC;EAGxDC,WAAWA,CAACC,MAAmC,EAAE;IACpD,IAAI,CAACC,MAAM,GAAGD,MAAM,CAACC,MAAM;EAC/B;EACA,MAAaC,KAAKA,CAAC;IACfC,KAAK;IACLC,MAAM;IACNC;EAC8B,CAAC,EAA+C;IAC9E,IAAIC,QAA6C;IACjD,IAAI;MACAA,QAAQ,GAAG,MAAM,IAAI,CAACL,MAAM,CAACM,MAAM,CAAC;QAChCJ,KAAK;QACLK,IAAI,EAAE;UACFC,KAAK,EAAE;YACHC,SAAS,EAAE,CAAC;UAChB,CAAC;UACDC,IAAI,EAAE;YACF,YAAY,EAAE;cACVC,KAAK,EAAE;YACX;UACJ,CAAC;UACDC,IAAI,EAAER,KAAK,GAAG,CAAC;UACfS,gBAAgB,EAAE,IAAI;UACtBC,YAAY,EAAEX,MAAM;UACpBY,OAAO,EAAE;QACb;MACJ,CAAC,CAAC;IACN,CAAC,CAAC,OAAOC,EAAE,EAAE;MACT;AACZ;AACA;MACY,IAAIrB,2BAA2B,CAACqB,EAAE,CAAC,EAAE;QACjC,IAAIC,OAAO,CAACC,GAAG,CAACC,KAAK,KAAK,MAAM,EAAE;UAC9BC,OAAO,CAACC,KAAK,CACTzB,OAAO,CAACoB,EAAE,EAAE;YACRM,KAAK,EAAE,CAAC;YACRC,UAAU,EAAE;UAChB,CAAC,CACL,CAAC;QACL;QACA,OAAO;UACHC,IAAI,EAAE,IAAI;UACVC,UAAU,EAAE,CAAC;UACbC,KAAK,EAAE;QACX,CAAC;MACL;MACAN,OAAO,CAACC,KAAK,CAAC,0CAA0C,EAAEL,EAAE,CAAC;MAC7D,MAAMA,EAAE;IACZ;IAEA,MAAM;MAAEW,IAAI;MAAEC;IAAM,CAAC,GAAGvB,QAAQ,CAACE,IAAI,CAACoB,IAAI;IAC1C,IAAIA,IAAI,CAACE,MAAM,KAAK,CAAC,EAAE;MACnB,OAAO;QACHL,IAAI,EAAE,IAAI;QACVrB,MAAM,EAAE2B,SAAS;QACjBL,UAAU,EAAEG,KAAK,CAACG,KAAK;QACvBL,KAAK,EAAE;MACX,CAAC;IACL;IAEA,MAAMM,YAAY,GAAGL,IAAI,CAACE,MAAM,GAAGzB,KAAK;IACxC,IAAI6B,UAAwC;IAC5C,IAAID,YAAY,EAAE;MACdL,IAAI,CAACO,GAAG,CAAC,CAAC;MACVD,UAAU,GAAGN,IAAI,CAACQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAEzB,IAAI;IAClC;IACA,MAAMgB,KAAK,GAAGC,IAAI,CAACS,MAAM,CAA2C,CAACC,UAAU,EAAEC,GAAG,KAAK;MACrF,MAAM,CAACC,EAAE,EAAEC,EAAE,CAAC,GAAGF,GAAG,CAACG,GAAG,CAACC,KAAK,CAAC,GAAG,CAAC;MACnC,IAAI,CAACH,EAAE,IAAI,CAACC,EAAE,EAAE;QACZ,OAAOH,UAAU;MACrB;MACAA,UAAU,CAACM,IAAI,CAAC;QACZJ,EAAE;QACFC,EAAE;QACFC,GAAG,EAAEH,GAAG,CAACG,GAAG;QACZvC,KAAK,EAAEoC,GAAG,CAACM;MACf,CAAC,CAAC;MAEF,OAAOP,UAAU;IACrB,CAAC,EAAE,EAAE,CAAC;IAEN,OAAO;MACHZ,UAAU,EAAEG,KAAK,CAACG,KAAK;MACvB5B,MAAM,EAAE8B,UAAU;MAClBT,IAAI,EAAE,CAACS,UAAU;MACjBP;IACJ,CAAC;EACL;AACJ","ignoreList":[]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { PrimitiveValue } from \"@webiny/api-
|
|
1
|
+
{"version":3,"names":[],"sources":["ElasticsearchFetcher.ts"],"sourcesContent":["import type { PrimitiveValue } from \"@webiny/api-opensearch/types.js\";\n\nexport interface IElasticsearchFetcherFetchResponseItem {\n PK: string;\n SK: string;\n _id: string;\n index: string;\n}\n\nexport interface IElasticsearchFetcherFetchParams {\n index: string;\n cursor?: PrimitiveValue[];\n limit: number;\n}\n\nexport interface IElasticsearchFetcherFetchResponse {\n done: boolean;\n totalCount: number;\n cursor?: PrimitiveValue[];\n items: IElasticsearchFetcherFetchResponseItem[];\n}\n\nexport interface IElasticsearchFetcher {\n fetch(params: IElasticsearchFetcherFetchParams): Promise<IElasticsearchFetcherFetchResponse>;\n}\n"],"mappings":"","ignoreList":[]}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { IManager } from "../../types.js";
|
|
2
|
-
import type { PrimitiveValue } from "@webiny/api-
|
|
2
|
+
import type { PrimitiveValue } from "@webiny/api-opensearch/types.js";
|
|
3
3
|
import type { IIndexManager } from "../../settings/types.js";
|
|
4
4
|
import type { IElasticsearchSynchronize } from "../../tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js";
|
|
5
5
|
import type { IElasticsearchFetcher } from "../../tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { IManager } from \"~/types.js\";\nimport type { PrimitiveValue } from \"@webiny/api-
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { IManager } from \"~/types.js\";\nimport type { PrimitiveValue } from \"@webiny/api-opensearch/types.js\";\nimport type { IIndexManager } from \"~/settings/types.js\";\nimport type { IElasticsearchSynchronize } from \"~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchSynchronize.js\";\nimport type { IElasticsearchFetcher } from \"~/tasks/dataSynchronization/elasticsearch/abstractions/ElasticsearchFetcher.js\";\nimport type { IGenericOutput } from \"@webiny/api-core/features/task/TaskService/index.js\";\nimport { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\n\nexport interface IDataSynchronizationInputValue {\n finished?: boolean;\n}\n\nexport interface IDataSynchronizationInputElasticsearchToDynamoDbValue\n extends IDataSynchronizationInputValue {\n index?: string;\n cursor?: PrimitiveValue[];\n}\n\nexport interface IDataSynchronizationInput {\n flow: \"elasticsearchToDynamoDb\";\n elasticsearchToDynamoDb?: IDataSynchronizationInputElasticsearchToDynamoDbValue;\n}\n\nexport type IDataSynchronizationOutput = IGenericOutput;\n\nexport type ISynchronizationRunResult =\n | TaskDefinition.ResultContinue<IDataSynchronizationInput>\n | TaskDefinition.ResultDone<IDataSynchronizationOutput>\n | TaskDefinition.ResultError\n | TaskDefinition.ResultAborted;\n\nexport interface ISynchronization {\n run(input: IDataSynchronizationInput): Promise<ISynchronizationRunResult>;\n}\n\nexport interface IElasticsearchSyncParams {\n manager: IDataSynchronizationManager;\n indexManager: IIndexManager;\n synchronize: IElasticsearchSynchronize;\n fetcher: IElasticsearchFetcher;\n}\n\nexport interface IElasticsearchSyncFactory {\n (params: IElasticsearchSyncParams): ISynchronization;\n}\n\nexport interface IFactories {\n /**\n * Delete all the records which are in the Elasticsearch but not in the Elasticsearch DynamoDB table.\n */\n elasticsearchToDynamoDb: IElasticsearchSyncFactory;\n}\n\nexport type IDataSynchronizationManager = IManager<\n IDataSynchronizationInput,\n IDataSynchronizationOutput\n>;\n"],"mappings":"","ignoreList":[]}
|
package/types.d.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { OpenSearchContext } from "@webiny/api-opensearch/types.js";
|
|
2
2
|
import type { Context as TasksContext } from "@webiny/tasks/types.js";
|
|
3
3
|
import type { DynamoDBDocument } from "@webiny/aws-sdk/client-dynamodb/index.js";
|
|
4
|
-
import type { Client,
|
|
4
|
+
import type { Client, createOpenSearchTable } from "@webiny/api-opensearch";
|
|
5
5
|
import type { BatchReadItem } from "@webiny/db-dynamodb/utils/batch/batchRead.js";
|
|
6
6
|
import type { IEntity } from "@webiny/db-dynamodb";
|
|
7
7
|
import type { GenericRecord } from "@webiny/api/types.js";
|
|
8
8
|
import { TaskController } from "@webiny/api-core/features/task/TaskController/index.js";
|
|
9
9
|
import { TaskDefinition } from "@webiny/api-core/features/task/TaskDefinition/index.js";
|
|
10
10
|
import { DbRegistry } from "./abstractions/DbRegistry.js";
|
|
11
|
-
export interface Context extends
|
|
11
|
+
export interface Context extends OpenSearchContext, TasksContext {
|
|
12
12
|
}
|
|
13
13
|
export interface IElasticsearchTaskConfig {
|
|
14
14
|
documentClient: DynamoDBDocument;
|
|
@@ -49,7 +49,7 @@ export interface IDynamoDbElasticsearchRecord {
|
|
|
49
49
|
export interface IManager<I extends TaskDefinition.TaskInput = TaskDefinition.TaskInput, O extends TaskDefinition.TaskOutput = TaskDefinition.TaskOutput> {
|
|
50
50
|
readonly documentClient: DynamoDBDocument;
|
|
51
51
|
readonly elasticsearch: Client;
|
|
52
|
-
readonly table: ReturnType<typeof
|
|
52
|
+
readonly table: ReturnType<typeof createOpenSearchTable>;
|
|
53
53
|
readonly controller: TaskController.Interface<I, O>;
|
|
54
54
|
readonly dbRegistry?: DbRegistry.Interface;
|
|
55
55
|
getEntity: (name: string) => IEntity;
|
package/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type {
|
|
1
|
+
{"version":3,"names":[],"sources":["types.ts"],"sourcesContent":["import type { OpenSearchContext } from \"@webiny/api-opensearch/types.js\";\nimport type { Context as TasksContext } from \"@webiny/tasks/types.js\";\nimport type { DynamoDBDocument } from \"@webiny/aws-sdk/client-dynamodb/index.js\";\nimport type { Client, createOpenSearchTable } from \"@webiny/api-opensearch\";\nimport type { BatchReadItem } from \"@webiny/db-dynamodb/utils/batch/batchRead.js\";\nimport type { IEntity } from \"@webiny/db-dynamodb\";\nimport type { GenericRecord } from \"@webiny/api/types.js\";\nimport { TaskController } from \"@webiny/api-core/features/task/TaskController/index.js\";\nimport { TaskDefinition } from \"@webiny/api-core/features/task/TaskDefinition/index.js\";\nimport { DbRegistry } from \"~/abstractions/DbRegistry.js\";\n\nexport interface Context extends OpenSearchContext, TasksContext {}\n\nexport interface IElasticsearchTaskConfig {\n documentClient: DynamoDBDocument;\n elasticsearchClient: Client;\n}\n\nexport interface IElasticsearchIndexingTaskValuesKeys {\n PK: string;\n SK: string;\n}\n\nexport interface IIndexSettingsValues {\n numberOfReplicas: number;\n refreshInterval: string;\n}\n\nexport interface IElasticsearchIndexingTaskValuesSettings {\n [key: string]: IIndexSettingsValues;\n}\n\nexport interface IElasticsearchIndexingTaskValues {\n matching?: string;\n limit?: number;\n keys?: IElasticsearchIndexingTaskValuesKeys;\n settings?: IElasticsearchIndexingTaskValuesSettings;\n}\n\nexport interface AugmentedError extends Error {\n data?: GenericRecord;\n [key: string]: any;\n}\n\nexport interface IDynamoDbElasticsearchRecord {\n PK: string;\n SK: string;\n GSI_TENANT: string;\n TYPE?: string;\n index: string;\n _et?: string;\n entity: string;\n data: GenericRecord;\n modified: string;\n}\n\nexport interface IManager<\n I extends TaskDefinition.TaskInput = TaskDefinition.TaskInput,\n O extends TaskDefinition.TaskOutput = TaskDefinition.TaskOutput\n> {\n readonly documentClient: DynamoDBDocument;\n readonly elasticsearch: Client;\n readonly table: ReturnType<typeof createOpenSearchTable>;\n readonly controller: TaskController.Interface<I, O>;\n readonly dbRegistry?: DbRegistry.Interface;\n getEntity: (name: string) => IEntity;\n read<T>(items: BatchReadItem[]): Promise<T[]>;\n}\n"],"mappings":"","ignoreList":[]}
|