@travetto/model-elasticsearch 2.1.5 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/package.json +5 -5
- package/src/config.ts +2 -2
- package/src/index-manager.ts +26 -23
- package/src/internal/query.ts +17 -6
- package/src/internal/schema.ts +15 -8
- package/src/service.ts +64 -49
package/README.md
CHANGED
|
@@ -8,9 +8,9 @@
|
|
|
8
8
|
npm install @travetto/model-elasticsearch
|
|
9
9
|
```
|
|
10
10
|
|
|
11
|
-
This module provides an [elasticsearch](https://elastic.co)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [elasticsearch](https://elastic.co). In development mode, [ElasticsearchModelService](https://github.com/travetto/travetto/tree/main/module/model-elasticsearch/src/service.ts#
|
|
11
|
+
This module provides an [elasticsearch](https://elastic.co)-based implementation of the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations."). This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [elasticsearch](https://elastic.co). In development mode, [ElasticsearchModelService](https://github.com/travetto/travetto/tree/main/module/model-elasticsearch/src/service.ts#L42) will also modify the [elasticsearch](https://elastic.co) schema in real time to minimize impact to development.
|
|
12
12
|
|
|
13
|
-
Supported
|
|
13
|
+
Supported features:
|
|
14
14
|
|
|
15
15
|
* [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/service/crud.ts#L11)
|
|
16
16
|
* [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/service/bulk.ts#L23)
|
|
@@ -94,14 +94,14 @@ export class ElasticsearchModelConfig {
|
|
|
94
94
|
};
|
|
95
95
|
|
|
96
96
|
/**
|
|
97
|
-
* Frequency of culling for
|
|
97
|
+
* Frequency of culling for cullable content
|
|
98
98
|
*/
|
|
99
99
|
cullRate?: number | TimeSpan;
|
|
100
100
|
|
|
101
101
|
/**
|
|
102
102
|
* Build final hosts
|
|
103
103
|
*/
|
|
104
|
-
postConstruct() {
|
|
104
|
+
postConstruct(): void {
|
|
105
105
|
console.debug('Constructed', { config: this });
|
|
106
106
|
this.hosts = this.hosts
|
|
107
107
|
.map(x => x.includes(':') ? x : `${x}:${this.port}`)
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@travetto/model-elasticsearch",
|
|
3
3
|
"displayName": "Elasticsearch Model Source",
|
|
4
|
-
"version": "2.
|
|
4
|
+
"version": "2.2.2",
|
|
5
5
|
"description": "Elasticsearch backing for the travetto model module, with real-time modeling support for Elasticsearch mappings.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"elasticsearch",
|
|
@@ -29,12 +29,12 @@
|
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
31
|
"@elastic/elasticsearch": "^7.17.0",
|
|
32
|
-
"@travetto/config": "^2.
|
|
33
|
-
"@travetto/model": "^2.
|
|
34
|
-
"@travetto/model-query": "2.
|
|
32
|
+
"@travetto/config": "^2.2.2",
|
|
33
|
+
"@travetto/model": "^2.2.2",
|
|
34
|
+
"@travetto/model-query": "2.2.2"
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
|
-
"@travetto/app": "^2.
|
|
37
|
+
"@travetto/app": "^2.2.2"
|
|
38
38
|
},
|
|
39
39
|
"publishConfig": {
|
|
40
40
|
"access": "public"
|
package/src/config.ts
CHANGED
|
@@ -47,14 +47,14 @@ export class ElasticsearchModelConfig {
|
|
|
47
47
|
};
|
|
48
48
|
|
|
49
49
|
/**
|
|
50
|
-
* Frequency of culling for
|
|
50
|
+
* Frequency of culling for cullable content
|
|
51
51
|
*/
|
|
52
52
|
cullRate?: number | TimeSpan;
|
|
53
53
|
|
|
54
54
|
/**
|
|
55
55
|
* Build final hosts
|
|
56
56
|
*/
|
|
57
|
-
postConstruct() {
|
|
57
|
+
postConstruct(): void {
|
|
58
58
|
console.debug('Constructed', { config: this });
|
|
59
59
|
this.hosts = this.hosts
|
|
60
60
|
.map(x => x.includes(':') ? x : `${x}:${this.port}`)
|
package/src/index-manager.ts
CHANGED
|
@@ -24,7 +24,7 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
24
24
|
this.#client = client;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
getStore(cls: Class) {
|
|
27
|
+
getStore(cls: Class): string {
|
|
28
28
|
return ModelRegistry.getStore(cls).toLowerCase().replace(/[^A-Za-z0-9_]+/g, '_');
|
|
29
29
|
}
|
|
30
30
|
|
|
@@ -32,7 +32,7 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
32
32
|
* Get namespaced index
|
|
33
33
|
* @param idx
|
|
34
34
|
*/
|
|
35
|
-
getNamespacedIndex(idx: string) {
|
|
35
|
+
getNamespacedIndex(idx: string): string {
|
|
36
36
|
if (this.config.namespace) {
|
|
37
37
|
return `${this.config.namespace}_${idx}`;
|
|
38
38
|
} else {
|
|
@@ -55,8 +55,9 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
55
55
|
/**
|
|
56
56
|
* Build alias mappings from the current state in the database
|
|
57
57
|
*/
|
|
58
|
-
async computeAliasMappings(force = false) {
|
|
58
|
+
async computeAliasMappings(force = false): Promise<void> {
|
|
59
59
|
if (force || !this.#indexToAlias.size) {
|
|
60
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
60
61
|
const { body: aliases } = (await this.#client.cat.aliases({
|
|
61
62
|
format: 'json'
|
|
62
63
|
})) as { body: { index: string, alias: string }[] };
|
|
@@ -75,7 +76,7 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
75
76
|
* @param cls
|
|
76
77
|
* @param alias
|
|
77
78
|
*/
|
|
78
|
-
async createIndex(cls: Class, alias = true) {
|
|
79
|
+
async createIndex(cls: Class, alias = true): Promise<string> {
|
|
79
80
|
const schema = ElasticsearchSchemaUtil.generateSourceSchema(cls, this.config.schemaConfig);
|
|
80
81
|
const ident = this.getIdentity(cls); // Already namespaced
|
|
81
82
|
const concreteIndex = `${ident.index}_${Date.now()}`;
|
|
@@ -93,8 +94,8 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
93
94
|
mappings: ElasticsearchSchemaUtil.MAJOR_VER < 7 ? { [ident.type!]: schema } : schema,
|
|
94
95
|
settings: this.config.indexCreate
|
|
95
96
|
});
|
|
96
|
-
} catch (
|
|
97
|
-
console.warn('Index already created', { index: ident.index, error:
|
|
97
|
+
} catch (err) {
|
|
98
|
+
console.warn('Index already created', { index: ident.index, error: err });
|
|
98
99
|
}
|
|
99
100
|
return concreteIndex;
|
|
100
101
|
}
|
|
@@ -102,23 +103,23 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
102
103
|
/**
|
|
103
104
|
* Build an index if missing
|
|
104
105
|
*/
|
|
105
|
-
async createIndexIfMissing(cls: Class) {
|
|
106
|
+
async createIndexIfMissing(cls: Class): Promise<void> {
|
|
106
107
|
cls = ModelRegistry.getBaseModel(cls);
|
|
107
108
|
const ident = this.getIdentity(cls);
|
|
108
109
|
try {
|
|
109
110
|
await this.#client.search(ident);
|
|
110
111
|
console.debug('Index already exists, not creating', ident);
|
|
111
|
-
} catch
|
|
112
|
+
} catch {
|
|
112
113
|
await this.createIndex(cls);
|
|
113
114
|
}
|
|
114
115
|
}
|
|
115
116
|
|
|
116
|
-
async createModel(cls: Class<ModelType>) {
|
|
117
|
+
async createModel(cls: Class<ModelType>): Promise<void> {
|
|
117
118
|
await this.createIndexIfMissing(cls);
|
|
118
119
|
await this.computeAliasMappings(true);
|
|
119
120
|
}
|
|
120
121
|
|
|
121
|
-
async exportModel(cls: Class<ModelType>) {
|
|
122
|
+
async exportModel(cls: Class<ModelType>): Promise<string> {
|
|
122
123
|
const schema = ElasticsearchSchemaUtil.generateSourceSchema(cls, this.config.schemaConfig);
|
|
123
124
|
const ident = this.getIdentity(cls); // Already namespaced
|
|
124
125
|
return `curl -XPOST $ES_HOST/${ident.index} -d '${JSON.stringify({
|
|
@@ -127,7 +128,7 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
127
128
|
})}'`;
|
|
128
129
|
}
|
|
129
130
|
|
|
130
|
-
async deleteModel(cls: Class<ModelType>) {
|
|
131
|
+
async deleteModel(cls: Class<ModelType>): Promise<void> {
|
|
131
132
|
const alias = this.getNamespacedIndex(this.getStore(cls));
|
|
132
133
|
if (this.#aliasToIndex.get(alias)) {
|
|
133
134
|
await this.#client.indices.delete({
|
|
@@ -140,23 +141,23 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
140
141
|
/**
|
|
141
142
|
* When the schema changes
|
|
142
143
|
*/
|
|
143
|
-
async changeSchema(cls: Class, change: SchemaChange) {
|
|
144
|
+
async changeSchema(cls: Class, change: SchemaChange): Promise<void> {
|
|
144
145
|
// Find which fields are gone
|
|
145
|
-
const removes = change.subs.reduce((acc, v) => {
|
|
146
|
+
const removes = change.subs.reduce<string[]>((acc, v) => {
|
|
146
147
|
acc.push(...v.fields
|
|
147
148
|
.filter(ev => ev.type === 'removing')
|
|
148
149
|
.map(ev => [...v.path.map(f => f.name), ev.prev!.name].join('.')));
|
|
149
150
|
return acc;
|
|
150
|
-
}, []
|
|
151
|
+
}, []);
|
|
151
152
|
|
|
152
153
|
// Find which types have changed
|
|
153
|
-
const fieldChanges = change.subs.reduce((acc, v) => {
|
|
154
|
+
const fieldChanges = change.subs.reduce<string[]>((acc, v) => {
|
|
154
155
|
acc.push(...v.fields
|
|
155
156
|
.filter(ev => ev.type === 'changed')
|
|
156
157
|
.filter(ev => ev.prev?.type !== ev.curr?.type)
|
|
157
158
|
.map(ev => [...v.path.map(f => f.name), ev.prev!.name].join('.')));
|
|
158
159
|
return acc;
|
|
159
|
-
}, []
|
|
160
|
+
}, []);
|
|
160
161
|
|
|
161
162
|
const { index, type } = this.getIdentity(cls);
|
|
162
163
|
|
|
@@ -169,8 +170,7 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
169
170
|
|
|
170
171
|
const allChange = removes.concat(fieldChanges);
|
|
171
172
|
|
|
172
|
-
|
|
173
|
-
await this.#client.reindex({
|
|
173
|
+
const reindexBody: Reindex = {
|
|
174
174
|
body: {
|
|
175
175
|
source: { index: curr },
|
|
176
176
|
dest: { index: next },
|
|
@@ -179,8 +179,11 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
179
179
|
source: allChange.map(x => `ctx._source.remove("${x}");`).join(' ') // Removing
|
|
180
180
|
}
|
|
181
181
|
},
|
|
182
|
-
|
|
183
|
-
}
|
|
182
|
+
wait_for_completion: true
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
// Reindex
|
|
186
|
+
await this.#client.reindex(reindexBody);
|
|
184
187
|
|
|
185
188
|
await Promise.all(Object.keys(aliases)
|
|
186
189
|
.map(x => this.#client.indices.delete({ index: x })));
|
|
@@ -197,13 +200,13 @@ export class IndexManager implements ModelStorageSupport {
|
|
|
197
200
|
}
|
|
198
201
|
}
|
|
199
202
|
|
|
200
|
-
async createStorage() {
|
|
201
|
-
//
|
|
203
|
+
async createStorage(): Promise<void> {
|
|
204
|
+
// Pre-create indexes if missing
|
|
202
205
|
console.debug('Create Storage', { idx: this.getNamespacedIndex('*') });
|
|
203
206
|
await this.computeAliasMappings(true);
|
|
204
207
|
}
|
|
205
208
|
|
|
206
|
-
async deleteStorage() {
|
|
209
|
+
async deleteStorage(): Promise<void> {
|
|
207
210
|
console.debug('Deleting storage', { idx: this.getNamespacedIndex('*') });
|
|
208
211
|
await this.#client.indices.delete({
|
|
209
212
|
index: this.getNamespacedIndex('*')
|
package/src/internal/query.ts
CHANGED
|
@@ -13,8 +13,11 @@ import { SchemaRegistry } from '@travetto/schema';
|
|
|
13
13
|
import { SearchResponse } from '../types';
|
|
14
14
|
import { EsSchemaConfig } from './types';
|
|
15
15
|
|
|
16
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
16
17
|
const has$And = (o: unknown): o is ({ $and: WhereClause<unknown>[] }) => !!o && '$and' in (o as object);
|
|
18
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
17
19
|
const has$Or = (o: unknown): o is ({ $or: WhereClause<unknown>[] }) => !!o && '$or' in (o as object);
|
|
20
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
18
21
|
const has$Not = (o: unknown): o is ({ $not: WhereClause<unknown> }) => !!o && '$not' in (o as object);
|
|
19
22
|
|
|
20
23
|
/**
|
|
@@ -27,10 +30,12 @@ export class ElasticsearchQueryUtil {
|
|
|
27
30
|
*/
|
|
28
31
|
static extractSimple<T>(o: T, path: string = ''): Record<string, unknown> {
|
|
29
32
|
const out: Record<string, unknown> = {};
|
|
33
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
30
34
|
const sub = o as Record<string, unknown>;
|
|
31
35
|
const keys = Object.keys(sub);
|
|
32
36
|
for (const key of keys) {
|
|
33
37
|
const subPath = `${path}${key}`;
|
|
38
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
34
39
|
if (Util.isPlainObject(sub[key]) && !Object.keys(sub[key] as object)[0].startsWith('$')) {
|
|
35
40
|
Object.assign(out, this.extractSimple(sub[key], `${subPath}.`));
|
|
36
41
|
} else {
|
|
@@ -43,12 +48,13 @@ export class ElasticsearchQueryUtil {
|
|
|
43
48
|
/**
|
|
44
49
|
* Build include/exclude from the select clause
|
|
45
50
|
*/
|
|
46
|
-
static getSelect<T>(clause: SelectClause<T>) {
|
|
51
|
+
static getSelect<T>(clause: SelectClause<T>): [string[], string[]] {
|
|
47
52
|
const simp = this.extractSimple(clause);
|
|
48
53
|
const include: string[] = [];
|
|
49
54
|
const exclude: string[] = [];
|
|
50
55
|
for (const k of Object.keys(simp)) {
|
|
51
56
|
const nk = k === 'id' ? '_id' : k;
|
|
57
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
52
58
|
const v = simp[k] as (1 | 0 | boolean);
|
|
53
59
|
if (v === 0 || v === false) {
|
|
54
60
|
exclude.push(nk);
|
|
@@ -62,10 +68,11 @@ export class ElasticsearchQueryUtil {
|
|
|
62
68
|
/**
|
|
63
69
|
* Build sort mechanism
|
|
64
70
|
*/
|
|
65
|
-
static getSort<T extends ModelType>(sort: SortClause<T>[] | IndexConfig<T>['fields']) {
|
|
71
|
+
static getSort<T extends ModelType>(sort: SortClause<T>[] | IndexConfig<T>['fields']): string[] {
|
|
66
72
|
return sort.map(x => {
|
|
67
73
|
const o = this.extractSimple(x);
|
|
68
74
|
const k = Object.keys(o)[0];
|
|
75
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
69
76
|
const v = o[k] as (boolean | -1 | 1);
|
|
70
77
|
if (v === 1 || v === true) {
|
|
71
78
|
return k;
|
|
@@ -82,6 +89,7 @@ export class ElasticsearchQueryUtil {
|
|
|
82
89
|
const items = [];
|
|
83
90
|
const schema = SchemaRegistry.getViewSchema(cls).schema;
|
|
84
91
|
|
|
92
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
85
93
|
for (const key of Object.keys(o) as (keyof typeof o)[]) {
|
|
86
94
|
const top = o[key];
|
|
87
95
|
const declaredSchema = schema[key];
|
|
@@ -154,6 +162,7 @@ export class ElasticsearchQueryUtil {
|
|
|
154
162
|
break;
|
|
155
163
|
}
|
|
156
164
|
case '$regex': {
|
|
165
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
157
166
|
const pattern = Util.toRegex(v as string);
|
|
158
167
|
if (pattern.source.startsWith('\\b') && pattern.source.endsWith('.*')) {
|
|
159
168
|
const textField = !pattern.flags.includes('i') && config && config.caseSensitive ?
|
|
@@ -180,6 +189,7 @@ export class ElasticsearchQueryUtil {
|
|
|
180
189
|
let dist = top.$maxDistance;
|
|
181
190
|
let unit = top.$unit ?? 'm';
|
|
182
191
|
if (unit === 'rad') {
|
|
192
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
183
193
|
dist = 6378.1 * (dist as number);
|
|
184
194
|
unit = 'km';
|
|
185
195
|
}
|
|
@@ -229,7 +239,7 @@ export class ElasticsearchQueryUtil {
|
|
|
229
239
|
* @param cls
|
|
230
240
|
* @param search
|
|
231
241
|
*/
|
|
232
|
-
static getSearchBody<T extends ModelType>(cls: Class<T>, search: Record<string, unknown>, checkExpiry = true) {
|
|
242
|
+
static getSearchBody<T extends ModelType>(cls: Class<T>, search: Record<string, unknown>, checkExpiry = true): { query?: Record<string, unknown> } {
|
|
233
243
|
const clauses = [];
|
|
234
244
|
if (search && Object.keys(search).length) {
|
|
235
245
|
clauses.push(search);
|
|
@@ -264,6 +274,7 @@ export class ElasticsearchQueryUtil {
|
|
|
264
274
|
QueryVerifier.verify(cls, query); // Verify
|
|
265
275
|
|
|
266
276
|
const search: Search = {
|
|
277
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
267
278
|
body: this.getSearchBody(cls, this.extractWhereQuery(cls, query.where as WhereClause<T>, config), checkExpiry)
|
|
268
279
|
};
|
|
269
280
|
|
|
@@ -301,7 +312,7 @@ export class ElasticsearchQueryUtil {
|
|
|
301
312
|
static cleanIdRemoval<T>(req: Search, results: SearchResponse<T>): T[] {
|
|
302
313
|
const out: T[] = [];
|
|
303
314
|
|
|
304
|
-
const toArr = <V>(x: V | V[] | undefined) => (x ? (Array.isArray(x) ? x : [x]) : []);
|
|
315
|
+
const toArr = <V>(x: V | V[] | undefined): V[] => (x ? (Array.isArray(x) ? x : [x]) : []);
|
|
305
316
|
|
|
306
317
|
// determine if id
|
|
307
318
|
const select = [
|
|
@@ -313,8 +324,8 @@ export class ElasticsearchQueryUtil {
|
|
|
313
324
|
for (const r of results.body.hits.hits) {
|
|
314
325
|
const obj = r._source;
|
|
315
326
|
if (includeId) {
|
|
316
|
-
// @
|
|
317
|
-
obj._id = r._id;
|
|
327
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
328
|
+
(obj as unknown as { _id: string })._id = r._id;
|
|
318
329
|
}
|
|
319
330
|
out.push(obj);
|
|
320
331
|
}
|
package/src/internal/schema.ts
CHANGED
|
@@ -22,6 +22,12 @@ type SchemaType = {
|
|
|
22
22
|
dynamic: boolean;
|
|
23
23
|
};
|
|
24
24
|
|
|
25
|
+
type UpdateScript = {
|
|
26
|
+
params: Record<string, unknown>;
|
|
27
|
+
lang: 'painless';
|
|
28
|
+
source: string;
|
|
29
|
+
};
|
|
30
|
+
|
|
25
31
|
/**
|
|
26
32
|
* Utils for ES Schema management
|
|
27
33
|
*/
|
|
@@ -32,10 +38,10 @@ export class ElasticsearchSchemaUtil {
|
|
|
32
38
|
/**
|
|
33
39
|
* Build the update script for a given object
|
|
34
40
|
*/
|
|
35
|
-
static generateUpdateScript(o: Record<string, unknown>, path: string = '', arr = false) {
|
|
41
|
+
static generateUpdateScript(o: Record<string, unknown>, path: string = '', arr = false): UpdateScript {
|
|
36
42
|
const ops: string[] = [];
|
|
37
|
-
const out = {
|
|
38
|
-
params: {}
|
|
43
|
+
const out: UpdateScript = {
|
|
44
|
+
params: {},
|
|
39
45
|
lang: 'painless',
|
|
40
46
|
source: ''
|
|
41
47
|
};
|
|
@@ -52,6 +58,7 @@ export class ElasticsearchSchemaUtil {
|
|
|
52
58
|
out.params[param] = o[x];
|
|
53
59
|
} else {
|
|
54
60
|
ops.push(`ctx._source.${prop} = ctx._source.${prop} == null ? [:] : ctx._source.${prop}`);
|
|
61
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
55
62
|
const sub = this.generateUpdateScript(o[x] as Record<string, unknown>, prop);
|
|
56
63
|
ops.push(sub.source);
|
|
57
64
|
Object.assign(out.params, sub.params);
|
|
@@ -65,7 +72,7 @@ export class ElasticsearchSchemaUtil {
|
|
|
65
72
|
/**
|
|
66
73
|
* Build one or more schemas depending on the polymorphic state
|
|
67
74
|
*/
|
|
68
|
-
static generateSourceSchema(cls: Class, config?: EsSchemaConfig) {
|
|
75
|
+
static generateSourceSchema(cls: Class, config?: EsSchemaConfig): SchemaType {
|
|
69
76
|
return ModelRegistry.get(cls).baseType ?
|
|
70
77
|
this.generateAllSourceSchema(cls, config) :
|
|
71
78
|
this.generateSingleSourceSchema(cls, config);
|
|
@@ -74,12 +81,12 @@ export class ElasticsearchSchemaUtil {
|
|
|
74
81
|
/**
|
|
75
82
|
* Generate all schemas
|
|
76
83
|
*/
|
|
77
|
-
static generateAllSourceSchema(cls: Class, config?: EsSchemaConfig) {
|
|
84
|
+
static generateAllSourceSchema(cls: Class, config?: EsSchemaConfig): SchemaType {
|
|
78
85
|
const allTypes = ModelRegistry.getClassesByBaseType(cls);
|
|
79
|
-
return allTypes.reduce((acc,
|
|
80
|
-
Util.deepAssign(acc, this.generateSingleSourceSchema(
|
|
86
|
+
return allTypes.reduce<SchemaType>((acc, schemaCls) => {
|
|
87
|
+
Util.deepAssign(acc, this.generateSingleSourceSchema(schemaCls, config));
|
|
81
88
|
return acc;
|
|
82
|
-
}, {}
|
|
89
|
+
}, { properties: {}, dynamic: false });
|
|
83
90
|
}
|
|
84
91
|
|
|
85
92
|
/**
|
package/src/service.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as es from '@elastic/elasticsearch';
|
|
2
|
-
import {
|
|
2
|
+
import { Search } from '@elastic/elasticsearch/api/requestParams';
|
|
3
3
|
|
|
4
4
|
import {
|
|
5
5
|
ModelCrudSupport, BulkOp, BulkResponse, ModelBulkSupport, ModelExpirySupport,
|
|
@@ -11,7 +11,7 @@ import { SchemaChange, DeepPartial } from '@travetto/schema';
|
|
|
11
11
|
import { Injectable } from '@travetto/di';
|
|
12
12
|
import {
|
|
13
13
|
ModelQuery, ModelQueryCrudSupport, ModelQueryFacetSupport,
|
|
14
|
-
ModelQuerySupport, PageableModelQuery, Query, ValidStringFields
|
|
14
|
+
ModelQuerySupport, PageableModelQuery, Query, SelectClause, ValidStringFields
|
|
15
15
|
} from '@travetto/model-query';
|
|
16
16
|
|
|
17
17
|
import { ModelCrudUtil } from '@travetto/model/src/internal/service/crud';
|
|
@@ -25,7 +25,7 @@ import { ModelQuerySuggestSupport } from '@travetto/model-query/src/service/sugg
|
|
|
25
25
|
import { ModelBulkUtil } from '@travetto/model/src/internal/service/bulk';
|
|
26
26
|
|
|
27
27
|
import { ElasticsearchModelConfig } from './config';
|
|
28
|
-
import {
|
|
28
|
+
import { EsBulkError } from './internal/types';
|
|
29
29
|
import { ElasticsearchQueryUtil } from './internal/query';
|
|
30
30
|
import { ElasticsearchSchemaUtil } from './internal/schema';
|
|
31
31
|
import { IndexManager } from './index-manager';
|
|
@@ -33,6 +33,8 @@ import { SearchResponse } from './types';
|
|
|
33
33
|
|
|
34
34
|
type WithId<T> = T & { _id?: string };
|
|
35
35
|
|
|
36
|
+
const isWithId = <T extends ModelType>(o: T): o is WithId<T> => !o && '_id' in o;
|
|
37
|
+
|
|
36
38
|
/**
|
|
37
39
|
* Elasticsearch model source.
|
|
38
40
|
*/
|
|
@@ -55,18 +57,20 @@ export class ElasticsearchModelService implements
|
|
|
55
57
|
async execSearch<T extends ModelType>(cls: Class<T>, search: Search<unknown>): Promise<SearchResponse<T>> {
|
|
56
58
|
const res = await this.client.search({
|
|
57
59
|
...this.manager.getIdentity(cls),
|
|
60
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
58
61
|
...search as Search<T>
|
|
59
62
|
});
|
|
63
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
60
64
|
return res as unknown as SearchResponse<T>;
|
|
61
65
|
}
|
|
62
66
|
|
|
63
67
|
/**
|
|
64
68
|
* Convert _id to id
|
|
65
69
|
*/
|
|
66
|
-
async postLoad<T extends ModelType>(cls: Class<T>, o: T) {
|
|
67
|
-
if (
|
|
68
|
-
|
|
69
|
-
delete
|
|
70
|
+
async postLoad<T extends ModelType>(cls: Class<T>, o: T): Promise<T> {
|
|
71
|
+
if (isWithId(o)) {
|
|
72
|
+
o.id = o._id!;
|
|
73
|
+
delete o._id;
|
|
70
74
|
}
|
|
71
75
|
|
|
72
76
|
o = await ModelCrudUtil.load(cls, o);
|
|
@@ -84,7 +88,7 @@ export class ElasticsearchModelService implements
|
|
|
84
88
|
}
|
|
85
89
|
}
|
|
86
90
|
|
|
87
|
-
async postConstruct(this: ElasticsearchModelService) {
|
|
91
|
+
async postConstruct(this: ElasticsearchModelService): Promise<void> {
|
|
88
92
|
this.client = new es.Client({
|
|
89
93
|
nodes: this.config.hosts,
|
|
90
94
|
...(this.config.options || {})
|
|
@@ -92,38 +96,39 @@ export class ElasticsearchModelService implements
|
|
|
92
96
|
await this.client.cluster.health({});
|
|
93
97
|
this.manager = new IndexManager(this.config, this.client);
|
|
94
98
|
|
|
99
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
95
100
|
await ModelStorageUtil.registerModelChangeListener(this.manager, this.constructor as Class);
|
|
96
101
|
ShutdownManager.onShutdown(this.constructor.ᚕid, () => this.client.close());
|
|
97
102
|
ModelExpiryUtil.registerCull(this);
|
|
98
103
|
}
|
|
99
104
|
|
|
100
|
-
createStorage() { return this.manager.createStorage(); }
|
|
101
|
-
deleteStorage() { return this.manager.deleteStorage(); }
|
|
102
|
-
createModel(cls: Class) { return this.manager.createModel(cls); }
|
|
103
|
-
exportModel(cls: Class) { return this.manager.exportModel(cls); }
|
|
104
|
-
deleteModel(cls: Class) { return this.manager.deleteModel(cls); }
|
|
105
|
-
changeSchema(cls: Class, change: SchemaChange) { return this.manager.changeSchema(cls, change); }
|
|
106
|
-
truncateModel(cls: Class) { return this.deleteByQuery(cls, {}).then(() => { }); }
|
|
105
|
+
createStorage(): Promise<void> { return this.manager.createStorage(); }
|
|
106
|
+
deleteStorage(): Promise<void> { return this.manager.deleteStorage(); }
|
|
107
|
+
createModel(cls: Class): Promise<void> { return this.manager.createModel(cls); }
|
|
108
|
+
exportModel(cls: Class): Promise<string> { return this.manager.exportModel(cls); }
|
|
109
|
+
deleteModel(cls: Class): Promise<void> { return this.manager.deleteModel(cls); }
|
|
110
|
+
changeSchema(cls: Class, change: SchemaChange): Promise<void> { return this.manager.changeSchema(cls, change); }
|
|
111
|
+
truncateModel(cls: Class): Promise<void> { return this.deleteByQuery(cls, {}).then(() => { }); }
|
|
107
112
|
|
|
108
|
-
uuid() {
|
|
113
|
+
uuid(): string {
|
|
109
114
|
return Util.uuid();
|
|
110
115
|
}
|
|
111
116
|
|
|
112
|
-
async get<T extends ModelType>(cls: Class<T>, id: string) {
|
|
117
|
+
async get<T extends ModelType>(cls: Class<T>, id: string): Promise<T> {
|
|
113
118
|
try {
|
|
114
119
|
const res = await this.client.get({ ...this.manager.getIdentity(cls), id });
|
|
115
120
|
return this.postLoad(cls, res.body._source);
|
|
116
|
-
} catch
|
|
121
|
+
} catch {
|
|
117
122
|
throw new NotFoundError(cls, id);
|
|
118
123
|
}
|
|
119
124
|
}
|
|
120
125
|
|
|
121
|
-
async delete<T extends ModelType>(cls: Class<T>, id: string) {
|
|
126
|
+
async delete<T extends ModelType>(cls: Class<T>, id: string): Promise<void> {
|
|
122
127
|
ModelCrudUtil.ensureNotSubType(cls);
|
|
123
128
|
|
|
124
129
|
try {
|
|
125
130
|
const { body: res } = await this.client.delete({
|
|
126
|
-
...this.manager.getIdentity(cls)
|
|
131
|
+
...this.manager.getIdentity(cls),
|
|
127
132
|
id,
|
|
128
133
|
refresh: true
|
|
129
134
|
});
|
|
@@ -131,7 +136,7 @@ export class ElasticsearchModelService implements
|
|
|
131
136
|
throw new NotFoundError(cls, id);
|
|
132
137
|
}
|
|
133
138
|
} catch (err) {
|
|
134
|
-
if (err.body && err.body.result === 'not_found') {
|
|
139
|
+
if (err && err instanceof es.errors.ResponseError && err.body && err.body.result === 'not_found') {
|
|
135
140
|
throw new NotFoundError(cls, id);
|
|
136
141
|
}
|
|
137
142
|
throw err;
|
|
@@ -144,7 +149,7 @@ export class ElasticsearchModelService implements
|
|
|
144
149
|
const id = clean.id;
|
|
145
150
|
|
|
146
151
|
await this.client.index({
|
|
147
|
-
...this.manager.getIdentity(cls)
|
|
152
|
+
...this.manager.getIdentity(cls),
|
|
148
153
|
id,
|
|
149
154
|
refresh: true,
|
|
150
155
|
body: clean
|
|
@@ -171,16 +176,16 @@ export class ElasticsearchModelService implements
|
|
|
171
176
|
await this.client.index({
|
|
172
177
|
...this.manager.getIdentity(cls),
|
|
173
178
|
id,
|
|
174
|
-
|
|
179
|
+
op_type: 'index',
|
|
175
180
|
refresh: true,
|
|
176
181
|
body: o
|
|
177
|
-
}
|
|
182
|
+
});
|
|
178
183
|
|
|
179
184
|
o.id = id;
|
|
180
185
|
return o;
|
|
181
186
|
}
|
|
182
187
|
|
|
183
|
-
async upsert<T extends ModelType>(cls: Class<T>, o: OptionalId<T>) {
|
|
188
|
+
async upsert<T extends ModelType>(cls: Class<T>, o: OptionalId<T>): Promise<T> {
|
|
184
189
|
ModelCrudUtil.ensureNotSubType(cls);
|
|
185
190
|
|
|
186
191
|
const item = await ModelCrudUtil.preStore(cls, o, this);
|
|
@@ -198,7 +203,7 @@ export class ElasticsearchModelService implements
|
|
|
198
203
|
return item;
|
|
199
204
|
}
|
|
200
205
|
|
|
201
|
-
async updatePartial<T extends ModelType>(cls: Class<T>, data: Partial<T> & { id: string }) {
|
|
206
|
+
async updatePartial<T extends ModelType>(cls: Class<T>, data: Partial<T> & { id: string }): Promise<T> {
|
|
202
207
|
ModelCrudUtil.ensureNotSubType(cls);
|
|
203
208
|
|
|
204
209
|
const script = ElasticsearchSchemaUtil.generateUpdateScript(data);
|
|
@@ -213,12 +218,12 @@ export class ElasticsearchModelService implements
|
|
|
213
218
|
body: {
|
|
214
219
|
script
|
|
215
220
|
}
|
|
216
|
-
}
|
|
221
|
+
});
|
|
217
222
|
|
|
218
223
|
return this.get(cls, id);
|
|
219
224
|
}
|
|
220
225
|
|
|
221
|
-
async * list<T extends ModelType>(cls: Class<T>) {
|
|
226
|
+
async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
|
|
222
227
|
let search: SearchResponse<T> = await this.execSearch(cls, {
|
|
223
228
|
scroll: '2m',
|
|
224
229
|
size: 100,
|
|
@@ -242,38 +247,42 @@ export class ElasticsearchModelService implements
|
|
|
242
247
|
}
|
|
243
248
|
}
|
|
244
249
|
|
|
245
|
-
async processBulk<T extends ModelType>(cls: Class<T>, operations: BulkOp<T>[]) {
|
|
250
|
+
async processBulk<T extends ModelType>(cls: Class<T>, operations: BulkOp<T>[]): Promise<BulkResponse<EsBulkError>> {
|
|
246
251
|
|
|
247
252
|
await ModelBulkUtil.preStore(cls, operations, this);
|
|
248
253
|
|
|
249
|
-
const body = operations.reduce((acc, op) => {
|
|
254
|
+
const body = operations.reduce<(T | Partial<Record<'delete' | 'create' | 'index' | 'update', { _index: string, _id?: string }>> | { doc: T })[]>((acc, op) => {
|
|
250
255
|
|
|
256
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
251
257
|
const esIdent = this.manager.getIdentity((op.upsert ?? op.delete ?? op.insert ?? op.update ?? { constructor: cls }).constructor as Class);
|
|
252
|
-
const ident = (ElasticsearchSchemaUtil.MAJOR_VER < 7 ?
|
|
258
|
+
const ident: { _index: string, _type?: unknown } = (ElasticsearchSchemaUtil.MAJOR_VER < 7 ?
|
|
253
259
|
{ _index: esIdent.index, _type: esIdent.type } :
|
|
254
|
-
{ _index: esIdent.index })
|
|
260
|
+
{ _index: esIdent.index });
|
|
255
261
|
|
|
256
262
|
if (op.delete) {
|
|
257
263
|
acc.push({ delete: { ...ident, _id: op.delete.id } });
|
|
258
264
|
} else if (op.insert) {
|
|
265
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
259
266
|
acc.push({ create: { ...ident, _id: op.insert.id } }, op.insert as T);
|
|
260
|
-
delete
|
|
267
|
+
delete op.insert.id;
|
|
261
268
|
} else if (op.upsert) {
|
|
269
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
262
270
|
acc.push({ index: { ...ident, _id: op.upsert.id } }, op.upsert as T);
|
|
263
|
-
delete
|
|
271
|
+
delete op.upsert.id;
|
|
264
272
|
} else if (op.update) {
|
|
265
273
|
acc.push({ update: { ...ident, _id: op.update.id } }, { doc: op.update });
|
|
266
|
-
|
|
274
|
+
// @ts-expect-error
|
|
275
|
+
delete op.update.id;
|
|
267
276
|
}
|
|
268
277
|
return acc;
|
|
269
|
-
}, []
|
|
278
|
+
}, []);
|
|
270
279
|
|
|
271
280
|
const { body: res } = await this.client.bulk({
|
|
272
281
|
body,
|
|
273
282
|
refresh: true
|
|
274
283
|
});
|
|
275
284
|
|
|
276
|
-
const out: BulkResponse = {
|
|
285
|
+
const out: BulkResponse<EsBulkError> = {
|
|
277
286
|
counts: {
|
|
278
287
|
delete: 0,
|
|
279
288
|
insert: 0,
|
|
@@ -282,14 +291,14 @@ export class ElasticsearchModelService implements
|
|
|
282
291
|
error: 0
|
|
283
292
|
},
|
|
284
293
|
insertedIds: new Map(),
|
|
285
|
-
errors: []
|
|
294
|
+
errors: []
|
|
286
295
|
};
|
|
287
296
|
|
|
288
297
|
type Count = keyof typeof out['counts'];
|
|
289
298
|
|
|
290
299
|
for (let i = 0; i < res.items.length; i++) {
|
|
291
300
|
const item = res.items[i];
|
|
292
|
-
const [k] = Object.keys
|
|
301
|
+
const [k] = Object.keys<Record<Count | 'create' | 'index', unknown>>(item);
|
|
293
302
|
const v = item[k]!;
|
|
294
303
|
if (v.error) {
|
|
295
304
|
out.errors.push(v.error);
|
|
@@ -317,12 +326,12 @@ export class ElasticsearchModelService implements
|
|
|
317
326
|
}
|
|
318
327
|
|
|
319
328
|
// Expiry
|
|
320
|
-
deleteExpired<T extends ModelType>(cls: Class<T>) {
|
|
329
|
+
deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
|
|
321
330
|
return ModelQueryExpiryUtil.deleteExpired(this, cls);
|
|
322
331
|
}
|
|
323
332
|
|
|
324
333
|
// Indexed
|
|
325
|
-
async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>) {
|
|
334
|
+
async getByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<T> {
|
|
326
335
|
const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body);
|
|
327
336
|
const res: SearchResponse<T> = await this.execSearch(cls, {
|
|
328
337
|
body: ElasticsearchQueryUtil.getSearchBody(cls,
|
|
@@ -336,7 +345,7 @@ export class ElasticsearchModelService implements
|
|
|
336
345
|
return this.postLoad(cls, res.body.hits.hits[0]._source);
|
|
337
346
|
}
|
|
338
347
|
|
|
339
|
-
async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>) {
|
|
348
|
+
async deleteByIndex<T extends ModelType>(cls: Class<T>, idx: string, body: DeepPartial<T>): Promise<void> {
|
|
340
349
|
const { key } = ModelIndexedUtil.computeIndexKey(cls, idx, body);
|
|
341
350
|
const res = await this.client.deleteByQuery({
|
|
342
351
|
index: this.manager.getIdentity(cls).index,
|
|
@@ -356,7 +365,7 @@ export class ElasticsearchModelService implements
|
|
|
356
365
|
return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
|
|
357
366
|
}
|
|
358
367
|
|
|
359
|
-
async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>) {
|
|
368
|
+
async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
|
|
360
369
|
const cfg = ModelRegistry.getIndex(cls, idx);
|
|
361
370
|
if (cfg.type === 'unique') {
|
|
362
371
|
throw new AppError('Cannot list on unique indices', 'data');
|
|
@@ -402,17 +411,19 @@ export class ElasticsearchModelService implements
|
|
|
402
411
|
|
|
403
412
|
async queryCount<T extends ModelType>(cls: Class<T>, query: Query<T>): Promise<number> {
|
|
404
413
|
const req = ElasticsearchQueryUtil.getSearchObject(cls, { ...query, limit: 0 }, this.config.schemaConfig);
|
|
405
|
-
const res = (await this.execSearch(cls, req)).body.hits.total
|
|
406
|
-
return
|
|
414
|
+
const res: number | { value: number } = (await this.execSearch(cls, req)).body.hits.total || { value: 0 };
|
|
415
|
+
return typeof res !== 'number' ? res.value : res;
|
|
407
416
|
}
|
|
408
417
|
|
|
409
418
|
// Query Crud
|
|
410
419
|
async deleteByQuery<T extends ModelType>(cls: Class<T>, query: ModelQuery<T> = {}): Promise<number> {
|
|
411
420
|
const { body: res } = await this.client.deleteByQuery({
|
|
421
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
422
|
+
body: undefined as unknown as {},
|
|
412
423
|
...this.manager.getIdentity(cls),
|
|
413
424
|
refresh: true,
|
|
414
425
|
...ElasticsearchQueryUtil.getSearchObject(cls, query, this.config.schemaConfig, false)
|
|
415
|
-
}
|
|
426
|
+
});
|
|
416
427
|
return res.deleted ?? 0;
|
|
417
428
|
}
|
|
418
429
|
|
|
@@ -425,6 +436,7 @@ export class ElasticsearchModelService implements
|
|
|
425
436
|
...this.manager.getIdentity(cls),
|
|
426
437
|
refresh: true,
|
|
427
438
|
body: {
|
|
439
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
428
440
|
query: (search.body as Record<string, unknown>).query,
|
|
429
441
|
script
|
|
430
442
|
}
|
|
@@ -444,9 +456,11 @@ export class ElasticsearchModelService implements
|
|
|
444
456
|
}
|
|
445
457
|
|
|
446
458
|
async suggestValues<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<string[]> {
|
|
459
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
460
|
+
const select: SelectClause<T> = { [field]: 1 } as SelectClause<T>;
|
|
461
|
+
|
|
447
462
|
const q = ModelQuerySuggestUtil.getSuggestQuery(cls, field, prefix, {
|
|
448
|
-
|
|
449
|
-
select: { [field]: 1 },
|
|
463
|
+
select,
|
|
450
464
|
...query
|
|
451
465
|
});
|
|
452
466
|
const search = ElasticsearchQueryUtil.getSearchObject(cls, q);
|
|
@@ -461,6 +475,7 @@ export class ElasticsearchModelService implements
|
|
|
461
475
|
|
|
462
476
|
const search = {
|
|
463
477
|
body: {
|
|
478
|
+
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
|
|
464
479
|
query: (q.body as Record<string, unknown>).query ?? { ['match_all']: {} },
|
|
465
480
|
aggs: { [field]: { terms: { field, size: 100 } } }
|
|
466
481
|
},
|
|
@@ -468,7 +483,7 @@ export class ElasticsearchModelService implements
|
|
|
468
483
|
};
|
|
469
484
|
|
|
470
485
|
const res = await this.execSearch(cls, search);
|
|
471
|
-
const { buckets } = res.body.aggregations[field
|
|
486
|
+
const { buckets } = res.body.aggregations[field];
|
|
472
487
|
const out = buckets.map(b => ({ key: b.key, count: b.doc_count }));
|
|
473
488
|
return out;
|
|
474
489
|
}
|