@travetto/model-mongo 7.0.0-rc.1 → 7.0.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -21
- package/package.json +5 -5
- package/src/config.ts +19 -19
- package/src/internal/util.ts +85 -55
- package/src/service.ts +84 -73
package/README.md
CHANGED
|
@@ -37,8 +37,8 @@ export class Init {
|
|
|
37
37
|
@InjectableFactory({
|
|
38
38
|
primary: true
|
|
39
39
|
})
|
|
40
|
-
static getModelSource(
|
|
41
|
-
return new MongoModelService(
|
|
40
|
+
static getModelSource(config: MongoModelConfig) {
|
|
41
|
+
return new MongoModelService(config);
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
44
|
```
|
|
@@ -84,9 +84,9 @@ export class MongoModelConfig {
|
|
|
84
84
|
options: mongo.MongoClientOptions = {};
|
|
85
85
|
|
|
86
86
|
/**
|
|
87
|
-
*
|
|
87
|
+
* Allow storage modification at runtime
|
|
88
88
|
*/
|
|
89
|
-
|
|
89
|
+
modifyStorage?: boolean;
|
|
90
90
|
|
|
91
91
|
/**
|
|
92
92
|
* Frequency of culling for cullable content
|
|
@@ -107,11 +107,11 @@ export class MongoModelConfig {
|
|
|
107
107
|
* Load all the ssl certs as needed
|
|
108
108
|
*/
|
|
109
109
|
async postConstruct(): Promise<void> {
|
|
110
|
-
const resolve = (file: string): Promise<string> => RuntimeResources.resolve(file).
|
|
110
|
+
const resolve = (file: string): Promise<string> => RuntimeResources.resolve(file).catch(() => file);
|
|
111
111
|
|
|
112
112
|
if (this.connectionString) {
|
|
113
113
|
const details = new URL(this.connectionString);
|
|
114
|
-
this.hosts ??= details.hostname.split(',').filter(
|
|
114
|
+
this.hosts ??= details.hostname.split(',').filter(host => !!host);
|
|
115
115
|
this.srvRecord ??= details.protocol === 'mongodb+srv:';
|
|
116
116
|
this.namespace ??= details.pathname.replace('/', '');
|
|
117
117
|
Object.assign(this.options, Object.fromEntries(details.searchParams.entries()));
|
|
@@ -131,24 +131,24 @@ export class MongoModelConfig {
|
|
|
131
131
|
this.hosts = ['localhost'];
|
|
132
132
|
}
|
|
133
133
|
|
|
134
|
-
const
|
|
135
|
-
if (
|
|
136
|
-
if (
|
|
137
|
-
|
|
134
|
+
const options = this.options;
|
|
135
|
+
if (options.ssl) {
|
|
136
|
+
if (options.cert) {
|
|
137
|
+
options.cert = await Promise.all([options.cert].flat(2).map(data => Buffer.isBuffer(data) ? data : resolve(data)));
|
|
138
138
|
}
|
|
139
|
-
if (
|
|
140
|
-
|
|
139
|
+
if (options.tlsCertificateKeyFile) {
|
|
140
|
+
options.tlsCertificateKeyFile = await resolve(options.tlsCertificateKeyFile);
|
|
141
141
|
}
|
|
142
|
-
if (
|
|
143
|
-
|
|
142
|
+
if (options.tlsCAFile) {
|
|
143
|
+
options.tlsCAFile = await resolve(options.tlsCAFile);
|
|
144
144
|
}
|
|
145
|
-
if (
|
|
146
|
-
|
|
145
|
+
if (options.tlsCRLFile) {
|
|
146
|
+
options.tlsCRLFile = await resolve(options.tlsCRLFile);
|
|
147
147
|
}
|
|
148
148
|
}
|
|
149
149
|
|
|
150
150
|
if (!Runtime.production) {
|
|
151
|
-
|
|
151
|
+
options.waitQueueTimeoutMS ??= TimeUtil.asMillis(1, 'd'); // Wait a day in dev mode
|
|
152
152
|
}
|
|
153
153
|
}
|
|
154
154
|
|
|
@@ -157,14 +157,14 @@ export class MongoModelConfig {
|
|
|
157
157
|
*/
|
|
158
158
|
get url(): string {
|
|
159
159
|
const hosts = this.hosts!
|
|
160
|
-
.map(
|
|
160
|
+
.map(host => (this.srvRecord || host.includes(':')) ? host : `${host}:${this.port ?? 27017}`)
|
|
161
161
|
.join(',');
|
|
162
|
-
const
|
|
162
|
+
const optionString = Object.entries(this.options).map(([key, value]) => `${key}=${value}`).join('&');
|
|
163
163
|
let creds = '';
|
|
164
164
|
if (this.username) {
|
|
165
|
-
creds = `${[this.username, this.password].filter(
|
|
165
|
+
creds = `${[this.username, this.password].filter(part => !!part).join(':')}@`;
|
|
166
166
|
}
|
|
167
|
-
const url = `mongodb${this.srvRecord ? '+srv' : ''}://${creds}${hosts}/${this.namespace}?${
|
|
167
|
+
const url = `mongodb${this.srvRecord ? '+srv' : ''}://${creds}${hosts}/${this.namespace}?${optionString}`;
|
|
168
168
|
return url;
|
|
169
169
|
}
|
|
170
170
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@travetto/model-mongo",
|
|
3
|
-
"version": "7.0.0-rc.
|
|
3
|
+
"version": "7.0.0-rc.3",
|
|
4
4
|
"description": "Mongo backing for the travetto model module.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"mongo",
|
|
@@ -25,10 +25,10 @@
|
|
|
25
25
|
"directory": "module/model-mongo"
|
|
26
26
|
},
|
|
27
27
|
"dependencies": {
|
|
28
|
-
"@travetto/cli": "^7.0.0-rc.
|
|
29
|
-
"@travetto/config": "^7.0.0-rc.
|
|
30
|
-
"@travetto/model": "^7.0.0-rc.
|
|
31
|
-
"@travetto/model-query": "^7.0.0-rc.
|
|
28
|
+
"@travetto/cli": "^7.0.0-rc.3",
|
|
29
|
+
"@travetto/config": "^7.0.0-rc.3",
|
|
30
|
+
"@travetto/model": "^7.0.0-rc.3",
|
|
31
|
+
"@travetto/model-query": "^7.0.0-rc.3",
|
|
32
32
|
"mongodb": "^7.0.0"
|
|
33
33
|
},
|
|
34
34
|
"travetto": {
|
package/src/config.ts
CHANGED
|
@@ -43,9 +43,9 @@ export class MongoModelConfig {
|
|
|
43
43
|
options: mongo.MongoClientOptions = {};
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
|
-
*
|
|
46
|
+
* Allow storage modification at runtime
|
|
47
47
|
*/
|
|
48
|
-
|
|
48
|
+
modifyStorage?: boolean;
|
|
49
49
|
|
|
50
50
|
/**
|
|
51
51
|
* Frequency of culling for cullable content
|
|
@@ -66,11 +66,11 @@ export class MongoModelConfig {
|
|
|
66
66
|
* Load all the ssl certs as needed
|
|
67
67
|
*/
|
|
68
68
|
async postConstruct(): Promise<void> {
|
|
69
|
-
const resolve = (file: string): Promise<string> => RuntimeResources.resolve(file).
|
|
69
|
+
const resolve = (file: string): Promise<string> => RuntimeResources.resolve(file).catch(() => file);
|
|
70
70
|
|
|
71
71
|
if (this.connectionString) {
|
|
72
72
|
const details = new URL(this.connectionString);
|
|
73
|
-
this.hosts ??= details.hostname.split(',').filter(
|
|
73
|
+
this.hosts ??= details.hostname.split(',').filter(host => !!host);
|
|
74
74
|
this.srvRecord ??= details.protocol === 'mongodb+srv:';
|
|
75
75
|
this.namespace ??= details.pathname.replace('/', '');
|
|
76
76
|
Object.assign(this.options, Object.fromEntries(details.searchParams.entries()));
|
|
@@ -90,24 +90,24 @@ export class MongoModelConfig {
|
|
|
90
90
|
this.hosts = ['localhost'];
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
const
|
|
94
|
-
if (
|
|
95
|
-
if (
|
|
96
|
-
|
|
93
|
+
const options = this.options;
|
|
94
|
+
if (options.ssl) {
|
|
95
|
+
if (options.cert) {
|
|
96
|
+
options.cert = await Promise.all([options.cert].flat(2).map(data => Buffer.isBuffer(data) ? data : resolve(data)));
|
|
97
97
|
}
|
|
98
|
-
if (
|
|
99
|
-
|
|
98
|
+
if (options.tlsCertificateKeyFile) {
|
|
99
|
+
options.tlsCertificateKeyFile = await resolve(options.tlsCertificateKeyFile);
|
|
100
100
|
}
|
|
101
|
-
if (
|
|
102
|
-
|
|
101
|
+
if (options.tlsCAFile) {
|
|
102
|
+
options.tlsCAFile = await resolve(options.tlsCAFile);
|
|
103
103
|
}
|
|
104
|
-
if (
|
|
105
|
-
|
|
104
|
+
if (options.tlsCRLFile) {
|
|
105
|
+
options.tlsCRLFile = await resolve(options.tlsCRLFile);
|
|
106
106
|
}
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
if (!Runtime.production) {
|
|
110
|
-
|
|
110
|
+
options.waitQueueTimeoutMS ??= TimeUtil.asMillis(1, 'd'); // Wait a day in dev mode
|
|
111
111
|
}
|
|
112
112
|
}
|
|
113
113
|
|
|
@@ -116,14 +116,14 @@ export class MongoModelConfig {
|
|
|
116
116
|
*/
|
|
117
117
|
get url(): string {
|
|
118
118
|
const hosts = this.hosts!
|
|
119
|
-
.map(
|
|
119
|
+
.map(host => (this.srvRecord || host.includes(':')) ? host : `${host}:${this.port ?? 27017}`)
|
|
120
120
|
.join(',');
|
|
121
|
-
const
|
|
121
|
+
const optionString = Object.entries(this.options).map(([key, value]) => `${key}=${value}`).join('&');
|
|
122
122
|
let creds = '';
|
|
123
123
|
if (this.username) {
|
|
124
|
-
creds = `${[this.username, this.password].filter(
|
|
124
|
+
creds = `${[this.username, this.password].filter(part => !!part).join(':')}@`;
|
|
125
125
|
}
|
|
126
|
-
const url = `mongodb${this.srvRecord ? '+srv' : ''}://${creds}${hosts}/${this.namespace}?${
|
|
126
|
+
const url = `mongodb${this.srvRecord ? '+srv' : ''}://${creds}${hosts}/${this.namespace}?${optionString}`;
|
|
127
127
|
return url;
|
|
128
128
|
}
|
|
129
129
|
}
|
package/src/internal/util.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
|
-
Binary, type CreateIndexesOptions, type Filter, type FindCursor, type IndexDirection, ObjectId, type WithId as MongoWithId
|
|
2
|
+
Binary, type CreateIndexesOptions, type Filter, type FindCursor, type IndexDirection, ObjectId, type WithId as MongoWithId,
|
|
3
|
+
type IndexDescriptionInfo
|
|
3
4
|
} from 'mongodb';
|
|
4
5
|
|
|
5
6
|
import { AppError, castTo, Class, toConcrete, TypedObject } from '@travetto/runtime';
|
|
@@ -7,9 +8,9 @@ import { type DistanceUnit, type PageableModelQuery, type WhereClause, ModelQuer
|
|
|
7
8
|
import type { ModelType, IndexField, IndexConfig } from '@travetto/model';
|
|
8
9
|
import { DataUtil, SchemaRegistryIndex, type Point } from '@travetto/schema';
|
|
9
10
|
|
|
10
|
-
const
|
|
11
|
+
const PointConcrete = toConcrete<Point>();
|
|
11
12
|
|
|
12
|
-
type
|
|
13
|
+
type IdxConfig = CreateIndexesOptions;
|
|
13
14
|
|
|
14
15
|
/**
|
|
15
16
|
* Converting units to various radians
|
|
@@ -31,21 +32,21 @@ export type PlainIdx = Record<string, -1 | 0 | 1>;
|
|
|
31
32
|
*/
|
|
32
33
|
export class MongoUtil {
|
|
33
34
|
|
|
34
|
-
static toIndex<T extends ModelType>(
|
|
35
|
+
static toIndex<T extends ModelType>(field: IndexField<T>): PlainIdx {
|
|
35
36
|
const keys = [];
|
|
36
|
-
while (typeof
|
|
37
|
-
const key = TypedObject.keys(
|
|
38
|
-
|
|
37
|
+
while (typeof field !== 'number' && typeof field !== 'boolean' && Object.keys(field)) {
|
|
38
|
+
const key = TypedObject.keys(field)[0];
|
|
39
|
+
field = castTo(field[key]);
|
|
39
40
|
keys.push(key);
|
|
40
41
|
}
|
|
41
|
-
const rf: number | boolean = castTo(
|
|
42
|
+
const rf: number | boolean = castTo(field);
|
|
42
43
|
return {
|
|
43
44
|
[keys.join('.')]: typeof rf === 'boolean' ? (rf ? 1 : 0) : castTo<-1 | 1 | 0>(rf)
|
|
44
45
|
};
|
|
45
46
|
}
|
|
46
47
|
|
|
47
|
-
static uuid(
|
|
48
|
-
return new Binary(Buffer.from(
|
|
48
|
+
static uuid(value: string): Binary {
|
|
49
|
+
return new Binary(Buffer.from(value.replaceAll('-', ''), 'hex'), Binary.SUBTYPE_UUID);
|
|
49
50
|
}
|
|
50
51
|
|
|
51
52
|
static idToString(id: string | ObjectId | Binary): string {
|
|
@@ -66,84 +67,84 @@ export class MongoUtil {
|
|
|
66
67
|
/**
|
|
67
68
|
* Build mongo where clause
|
|
68
69
|
*/
|
|
69
|
-
static extractWhereClause<T>(cls: Class<T>,
|
|
70
|
-
if (ModelQueryUtil.has$And(
|
|
71
|
-
return { $and:
|
|
72
|
-
} else if (ModelQueryUtil.has$Or(
|
|
73
|
-
return { $or:
|
|
74
|
-
} else if (ModelQueryUtil.has$Not(
|
|
75
|
-
return { $nor: [this.extractWhereClause<T>(cls,
|
|
70
|
+
static extractWhereClause<T>(cls: Class<T>, clause: WhereClause<T>): Record<string, unknown> {
|
|
71
|
+
if (ModelQueryUtil.has$And(clause)) {
|
|
72
|
+
return { $and: clause.$and.map(item => this.extractWhereClause<T>(cls, item)) };
|
|
73
|
+
} else if (ModelQueryUtil.has$Or(clause)) {
|
|
74
|
+
return { $or: clause.$or.map(item => this.extractWhereClause<T>(cls, item)) };
|
|
75
|
+
} else if (ModelQueryUtil.has$Not(clause)) {
|
|
76
|
+
return { $nor: [this.extractWhereClause<T>(cls, clause.$not)] };
|
|
76
77
|
} else {
|
|
77
|
-
return this.extractSimple(cls,
|
|
78
|
+
return this.extractSimple(cls, clause);
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
81
|
|
|
81
82
|
/**/
|
|
82
|
-
static extractSimple<T>(base: Class<T> | undefined,
|
|
83
|
+
static extractSimple<T>(base: Class<T> | undefined, item: Record<string, unknown>, path: string = '', recursive: boolean = true): Record<string, unknown> {
|
|
83
84
|
const fields = base ? SchemaRegistryIndex.getOptional(base)?.getFields() : undefined;
|
|
84
85
|
const out: Record<string, unknown> = {};
|
|
85
|
-
const sub =
|
|
86
|
+
const sub = item;
|
|
86
87
|
const keys = Object.keys(sub);
|
|
87
88
|
for (const key of keys) {
|
|
88
89
|
const subpath = `${path}${key}`;
|
|
89
|
-
const
|
|
90
|
+
const value: Record<string, unknown> = castTo(sub[key]);
|
|
90
91
|
const subField = fields?.[key];
|
|
91
92
|
|
|
92
|
-
const isPlain =
|
|
93
|
-
const firstKey = isPlain ? Object.keys(
|
|
93
|
+
const isPlain = value && DataUtil.isPlainObject(value);
|
|
94
|
+
const firstKey = isPlain ? Object.keys(value)[0] : '';
|
|
94
95
|
|
|
95
96
|
if (subpath === 'id') {
|
|
96
97
|
if (!firstKey) {
|
|
97
|
-
out._id = Array.isArray(
|
|
98
|
+
out._id = Array.isArray(value) ? value.map(subValue => this.uuid(subValue)) : this.uuid(`${value}`);
|
|
98
99
|
} else if (firstKey === '$in' || firstKey === '$nin' || firstKey === '$eq' || firstKey === '$ne') {
|
|
99
|
-
const temp =
|
|
100
|
-
out._id = { [firstKey]: Array.isArray(temp) ? temp.map(
|
|
100
|
+
const temp = value[firstKey];
|
|
101
|
+
out._id = { [firstKey]: Array.isArray(temp) ? temp.map(subValue => this.uuid(subValue)) : this.uuid(`${temp}`) };
|
|
101
102
|
} else {
|
|
102
103
|
throw new AppError('Invalid id query');
|
|
103
104
|
}
|
|
104
|
-
} else if ((isPlain && !firstKey.startsWith('$')) ||
|
|
105
|
+
} else if ((isPlain && !firstKey.startsWith('$')) || value?.constructor?.Ⲑid) {
|
|
105
106
|
if (recursive) {
|
|
106
|
-
Object.assign(out, this.extractSimple(subField?.type,
|
|
107
|
+
Object.assign(out, this.extractSimple(subField?.type, value, `${subpath}.`, recursive));
|
|
107
108
|
} else {
|
|
108
|
-
out[subpath] =
|
|
109
|
+
out[subpath] = value;
|
|
109
110
|
}
|
|
110
111
|
} else {
|
|
111
112
|
if (firstKey === '$gt' || firstKey === '$lt' || firstKey === '$gte' || firstKey === '$lte') {
|
|
112
|
-
for (const [sk, sv] of Object.entries(
|
|
113
|
-
|
|
113
|
+
for (const [sk, sv] of Object.entries(value)) {
|
|
114
|
+
value[sk] = ModelQueryUtil.resolveComparator(sv);
|
|
114
115
|
}
|
|
115
116
|
} else if (firstKey === '$exists' && subField?.array) {
|
|
116
|
-
const exists =
|
|
117
|
+
const exists = value.$exists;
|
|
117
118
|
if (!exists) {
|
|
118
|
-
delete
|
|
119
|
-
|
|
119
|
+
delete value.$exists;
|
|
120
|
+
value.$in = [null, []];
|
|
120
121
|
} else {
|
|
121
|
-
|
|
122
|
-
|
|
122
|
+
value.$exists = true;
|
|
123
|
+
value.$nin = [null, []];
|
|
123
124
|
}
|
|
124
125
|
} else if (firstKey === '$regex') {
|
|
125
|
-
|
|
126
|
-
} else if (firstKey && '$near' in
|
|
127
|
-
const dist: number = castTo(
|
|
128
|
-
const distance = dist / RADIANS_TO[(castTo<DistanceUnit>(
|
|
129
|
-
|
|
130
|
-
delete
|
|
131
|
-
} else if (firstKey && '$geoWithin' in
|
|
132
|
-
const coords: [number, number][] = castTo(
|
|
126
|
+
value.$regex = DataUtil.toRegex(castTo(value.$regex));
|
|
127
|
+
} else if (firstKey && '$near' in value) {
|
|
128
|
+
const dist: number = castTo(value.$maxDistance);
|
|
129
|
+
const distance = dist / RADIANS_TO[(castTo<DistanceUnit>(value.$unit) ?? 'km')];
|
|
130
|
+
value.$maxDistance = distance;
|
|
131
|
+
delete value.$unit;
|
|
132
|
+
} else if (firstKey && '$geoWithin' in value) {
|
|
133
|
+
const coords: [number, number][] = castTo(value.$geoWithin);
|
|
133
134
|
const first = coords[0];
|
|
134
135
|
const last = coords.at(-1)!;
|
|
135
136
|
// Connect if not
|
|
136
137
|
if (first[0] !== last[0] || first[1] !== last[1]) {
|
|
137
138
|
coords.push(first);
|
|
138
139
|
}
|
|
139
|
-
|
|
140
|
+
value.$geoWithin = {
|
|
140
141
|
$geometry: {
|
|
141
142
|
type: 'Polygon',
|
|
142
143
|
coordinates: [coords]
|
|
143
144
|
}
|
|
144
145
|
};
|
|
145
146
|
}
|
|
146
|
-
out[subpath === 'id' ? '_id' : subpath] =
|
|
147
|
+
out[subpath === 'id' ? '_id' : subpath] = value;
|
|
147
148
|
}
|
|
148
149
|
}
|
|
149
150
|
return out;
|
|
@@ -153,11 +154,11 @@ export class MongoUtil {
|
|
|
153
154
|
const out: BasicIdx[] = [];
|
|
154
155
|
const textFields: string[] = [];
|
|
155
156
|
SchemaRegistryIndex.visitFields(cls, (field, path) => {
|
|
156
|
-
if (field.type ===
|
|
157
|
-
const name = [...path, field].map(
|
|
157
|
+
if (field.type === PointConcrete) {
|
|
158
|
+
const name = [...path, field].map(schema => schema.name).join('.');
|
|
158
159
|
out.push({ [name]: '2d' });
|
|
159
160
|
} else if (field.specifiers?.includes('text') && (field.specifiers?.includes('long') || field.specifiers.includes('search'))) {
|
|
160
|
-
const name = [...path, field].map(
|
|
161
|
+
const name = [...path, field].map(schema => schema.name).join('.');
|
|
161
162
|
textFields.push(name);
|
|
162
163
|
}
|
|
163
164
|
});
|
|
@@ -173,17 +174,17 @@ export class MongoUtil {
|
|
|
173
174
|
|
|
174
175
|
static getPlainIndex(idx: IndexConfig<ModelType>): PlainIdx {
|
|
175
176
|
let out: PlainIdx = {};
|
|
176
|
-
for (const
|
|
177
|
-
out = Object.assign(out,
|
|
177
|
+
for (const config of idx.fields.map(value => this.toIndex(value))) {
|
|
178
|
+
out = Object.assign(out, config);
|
|
178
179
|
}
|
|
179
180
|
return out;
|
|
180
181
|
}
|
|
181
182
|
|
|
182
|
-
static getIndices<T extends ModelType>(cls: Class<T>, indices: IndexConfig<ModelType>[] = []): [BasicIdx,
|
|
183
|
+
static getIndices<T extends ModelType>(cls: Class<T>, indices: IndexConfig<ModelType>[] = []): [BasicIdx, IdxConfig][] {
|
|
183
184
|
return [
|
|
184
185
|
...indices.map(idx => [this.getPlainIndex(idx), (idx.type === 'unique' ? { unique: true } : {})] as const),
|
|
185
|
-
...this.getExtraIndices(cls).map((
|
|
186
|
-
].map(
|
|
186
|
+
...this.getExtraIndices(cls).map((idx) => [idx, {}] as const)
|
|
187
|
+
].map(idx => [...idx]);
|
|
187
188
|
}
|
|
188
189
|
|
|
189
190
|
static prepareCursor<T extends ModelType>(cls: Class<T>, cursor: FindCursor<T | MongoWithId<T>>, query: PageableModelQuery<T>): FindCursor<T> {
|
|
@@ -201,7 +202,7 @@ export class MongoUtil {
|
|
|
201
202
|
}
|
|
202
203
|
|
|
203
204
|
if (query.sort) {
|
|
204
|
-
cursor = cursor.sort(Object.assign({}, ...query.sort.map(
|
|
205
|
+
cursor = cursor.sort(Object.assign({}, ...query.sort.map(item => this.extractSimple(cls, item))));
|
|
205
206
|
}
|
|
206
207
|
|
|
207
208
|
cursor = cursor.limit(Math.trunc(query.limit ?? 200));
|
|
@@ -212,4 +213,33 @@ export class MongoUtil {
|
|
|
212
213
|
|
|
213
214
|
return castTo(cursor);
|
|
214
215
|
}
|
|
216
|
+
|
|
217
|
+
static isIndexChanged(existing: IndexDescriptionInfo, [pendingKey, pendingOptions]: [BasicIdx, CreateIndexesOptions]): boolean {
|
|
218
|
+
// Config changed
|
|
219
|
+
if (
|
|
220
|
+
!!existing.unique !== !!pendingOptions.unique ||
|
|
221
|
+
!!existing.sparse !== !!pendingOptions.sparse ||
|
|
222
|
+
existing.expireAfterSeconds !== pendingOptions.expireAfterSeconds ||
|
|
223
|
+
existing.bucketSize !== pendingOptions.bucketSize
|
|
224
|
+
) {
|
|
225
|
+
return true;
|
|
226
|
+
}
|
|
227
|
+
const pendingKeySet = new Set(Object.keys(pendingKey));
|
|
228
|
+
const existingKeySet = new Set(Object.keys(existing.key));
|
|
229
|
+
|
|
230
|
+
if (pendingKeySet.size !== existingKeySet.size) {
|
|
231
|
+
return true;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const overlap = pendingKeySet.intersection(existingKeySet);
|
|
235
|
+
if (overlap.size !== pendingKeySet.size) {
|
|
236
|
+
return true;
|
|
237
|
+
}
|
|
238
|
+
for (const key of overlap) {
|
|
239
|
+
if (existing.key[key] !== pendingKey[key]) {
|
|
240
|
+
return false;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
215
245
|
}
|
package/src/service.ts
CHANGED
|
@@ -3,12 +3,12 @@ import { pipeline } from 'node:stream/promises';
|
|
|
3
3
|
import {
|
|
4
4
|
type Db, GridFSBucket, MongoClient, type GridFSFile, type Collection,
|
|
5
5
|
type ObjectId, type Binary, type RootFilterOperators, type Filter,
|
|
6
|
-
type WithId as MongoWithId
|
|
6
|
+
type WithId as MongoWithId,
|
|
7
7
|
} from 'mongodb';
|
|
8
8
|
|
|
9
9
|
import {
|
|
10
10
|
ModelRegistryIndex, ModelType, OptionalId, ModelCrudSupport, ModelStorageSupport,
|
|
11
|
-
ModelExpirySupport, ModelBulkSupport, ModelIndexedSupport,
|
|
11
|
+
ModelExpirySupport, ModelBulkSupport, ModelIndexedSupport, BulkOperation, BulkResponse,
|
|
12
12
|
NotFoundError, ExistsError, ModelBlobSupport,
|
|
13
13
|
ModelCrudUtil, ModelIndexedUtil, ModelStorageUtil, ModelExpiryUtil, ModelBulkUtil, ModelBlobUtil,
|
|
14
14
|
} from '@travetto/model';
|
|
@@ -106,7 +106,7 @@ export class MongoModelService implements
|
|
|
106
106
|
bucketName: ModelBlobNamespace,
|
|
107
107
|
writeConcern: { w: 1 }
|
|
108
108
|
});
|
|
109
|
-
await ModelStorageUtil.
|
|
109
|
+
await ModelStorageUtil.storageInitialization(this);
|
|
110
110
|
ShutdownManager.onGracefulShutdown(() => this.client.close());
|
|
111
111
|
ModelExpiryUtil.registerCull(this);
|
|
112
112
|
}
|
|
@@ -126,23 +126,34 @@ export class MongoModelService implements
|
|
|
126
126
|
await this.#db.dropDatabase();
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
-
async
|
|
129
|
+
async upsertModel(cls: Class): Promise<void> {
|
|
130
130
|
const col = await this.getStore(cls);
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
console.debug('Creating indexes', { indices: creating });
|
|
134
|
-
for (const el of creating) {
|
|
135
|
-
await col.createIndex(...el);
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
131
|
+
const indices = MongoUtil.getIndices(cls, ModelRegistryIndex.getConfig(cls).indices);
|
|
132
|
+
const existingIndices = (await col.indexes().catch(() => [])).filter(idx => idx.name !== '_id_');
|
|
139
133
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
}
|
|
134
|
+
const pendingMap = Object.fromEntries(indices.map(pair => [pair[1].name!, pair]));
|
|
135
|
+
const existingMap = Object.fromEntries(existingIndices.map(idx => [idx.name!, idx.key]));
|
|
143
136
|
|
|
144
|
-
|
|
145
|
-
|
|
137
|
+
for (const idx of existingIndices) {
|
|
138
|
+
if (!idx.name) {
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
const pending = pendingMap[idx.name];
|
|
142
|
+
if (!pending) {
|
|
143
|
+
console.debug('Deleting index', { indices: idx.name });
|
|
144
|
+
await col.dropIndex(idx.name);
|
|
145
|
+
} else if (MongoUtil.isIndexChanged(idx, pending)) {
|
|
146
|
+
console.debug('Updating index', { indices: idx.name });
|
|
147
|
+
await col.dropIndex(idx.name);
|
|
148
|
+
await col.createIndex(...pending);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
for (const [name, idx] of Object.entries(pendingMap)) {
|
|
152
|
+
if (!existingMap[name]) {
|
|
153
|
+
console.debug('Creating index', { indices: name });
|
|
154
|
+
await col.createIndex(...idx);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
146
157
|
}
|
|
147
158
|
|
|
148
159
|
async truncateModel<T extends ModelType>(cls: Class<T>): Promise<void> {
|
|
@@ -158,7 +169,7 @@ export class MongoModelService implements
|
|
|
158
169
|
* Get mongo collection
|
|
159
170
|
*/
|
|
160
171
|
async getStore<T extends ModelType>(cls: Class<T>): Promise<Collection<T>> {
|
|
161
|
-
return this.#db.collection(ModelRegistryIndex.getStoreName(cls)
|
|
172
|
+
return this.#db.collection(ModelRegistryIndex.getStoreName(cls));
|
|
162
173
|
}
|
|
163
174
|
|
|
164
175
|
// Crud
|
|
@@ -208,11 +219,11 @@ export class MongoModelService implements
|
|
|
208
219
|
{ $set: cleaned },
|
|
209
220
|
{ upsert: true }
|
|
210
221
|
);
|
|
211
|
-
} catch (
|
|
212
|
-
if (
|
|
222
|
+
} catch (error) {
|
|
223
|
+
if (error instanceof Error && error.message.includes('duplicate key error')) {
|
|
213
224
|
throw new ExistsError(cls, id);
|
|
214
225
|
} else {
|
|
215
|
-
throw
|
|
226
|
+
throw error;
|
|
216
227
|
}
|
|
217
228
|
}
|
|
218
229
|
return this.postUpdate(cleaned, id);
|
|
@@ -226,13 +237,13 @@ export class MongoModelService implements
|
|
|
226
237
|
|
|
227
238
|
const operation: Partial<T> = castTo(Object
|
|
228
239
|
.entries(simple)
|
|
229
|
-
.reduce<Partial<Record<'$unset' | '$set', Record<string, unknown>>>>((
|
|
230
|
-
if (
|
|
231
|
-
(
|
|
240
|
+
.reduce<Partial<Record<'$unset' | '$set', Record<string, unknown>>>>((document, [key, value]) => {
|
|
241
|
+
if (value === null || value === undefined) {
|
|
242
|
+
(document.$unset ??= {})[key] = value;
|
|
232
243
|
} else {
|
|
233
|
-
(
|
|
244
|
+
(document.$set ??= {})[key] = value;
|
|
234
245
|
}
|
|
235
|
-
return
|
|
246
|
+
return document;
|
|
236
247
|
}, {}));
|
|
237
248
|
|
|
238
249
|
const id = item.id;
|
|
@@ -261,12 +272,12 @@ export class MongoModelService implements
|
|
|
261
272
|
async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
|
|
262
273
|
const store = await this.getStore(cls);
|
|
263
274
|
const cursor = store.find(this.getWhereFilter(cls, {}), { timeout: true }).batchSize(100);
|
|
264
|
-
for await (const
|
|
275
|
+
for await (const item of cursor) {
|
|
265
276
|
try {
|
|
266
|
-
yield await this.postLoad(cls,
|
|
267
|
-
} catch (
|
|
268
|
-
if (!(
|
|
269
|
-
throw
|
|
277
|
+
yield await this.postLoad(cls, item);
|
|
278
|
+
} catch (error) {
|
|
279
|
+
if (!(error instanceof NotFoundError)) {
|
|
280
|
+
throw error;
|
|
270
281
|
}
|
|
271
282
|
}
|
|
272
283
|
}
|
|
@@ -313,7 +324,7 @@ export class MongoModelService implements
|
|
|
313
324
|
}
|
|
314
325
|
|
|
315
326
|
// Bulk
|
|
316
|
-
async processBulk<T extends ModelType>(cls: Class<T>, operations:
|
|
327
|
+
async processBulk<T extends ModelType>(cls: Class<T>, operations: BulkOperation<T>[]): Promise<BulkResponse<{ index: number }>> {
|
|
317
328
|
const out: BulkResponse<{ index: number }> = {
|
|
318
329
|
errors: [],
|
|
319
330
|
counts: {
|
|
@@ -336,26 +347,26 @@ export class MongoModelService implements
|
|
|
336
347
|
|
|
337
348
|
out.insertedIds = new Map([...upsertedIds.entries(), ...insertedIds.entries()]);
|
|
338
349
|
|
|
339
|
-
for (const
|
|
340
|
-
if (
|
|
341
|
-
this.preUpdate(
|
|
342
|
-
bulk.insert(
|
|
343
|
-
} else if (
|
|
344
|
-
const id = this.preUpdate(
|
|
345
|
-
bulk.find({ _id: MongoUtil.uuid(id!) }).upsert().updateOne({ $set:
|
|
346
|
-
} else if (
|
|
347
|
-
const id = this.preUpdate(
|
|
348
|
-
bulk.find({ _id: MongoUtil.uuid(id) }).update({ $set:
|
|
349
|
-
} else if (
|
|
350
|
-
bulk.find({ _id: MongoUtil.uuid(
|
|
350
|
+
for (const operation of operations) {
|
|
351
|
+
if (operation.insert) {
|
|
352
|
+
this.preUpdate(operation.insert);
|
|
353
|
+
bulk.insert(operation.insert);
|
|
354
|
+
} else if (operation.upsert) {
|
|
355
|
+
const id = this.preUpdate(operation.upsert);
|
|
356
|
+
bulk.find({ _id: MongoUtil.uuid(id!) }).upsert().updateOne({ $set: operation.upsert });
|
|
357
|
+
} else if (operation.update) {
|
|
358
|
+
const id = this.preUpdate(operation.update);
|
|
359
|
+
bulk.find({ _id: MongoUtil.uuid(id) }).update({ $set: operation.update });
|
|
360
|
+
} else if (operation.delete) {
|
|
361
|
+
bulk.find({ _id: MongoUtil.uuid(operation.delete.id) }).deleteOne();
|
|
351
362
|
}
|
|
352
363
|
}
|
|
353
364
|
|
|
354
365
|
const result = await bulk.execute({});
|
|
355
366
|
|
|
356
367
|
// Restore all ids
|
|
357
|
-
for (const
|
|
358
|
-
const core =
|
|
368
|
+
for (const operation of operations) {
|
|
369
|
+
const core = operation.insert ?? operation.upsert ?? operation.update;
|
|
359
370
|
if (core) {
|
|
360
371
|
this.postUpdate(asFull(core));
|
|
361
372
|
}
|
|
@@ -367,17 +378,17 @@ export class MongoModelService implements
|
|
|
367
378
|
|
|
368
379
|
if (out.counts) {
|
|
369
380
|
out.counts.delete = result.deletedCount;
|
|
370
|
-
out.counts.update = operations.filter(
|
|
381
|
+
out.counts.update = operations.filter(item => item.update).length;
|
|
371
382
|
out.counts.insert = result.insertedCount;
|
|
372
|
-
out.counts.upsert = operations.filter(
|
|
383
|
+
out.counts.upsert = operations.filter(item => item.upsert).length;
|
|
373
384
|
}
|
|
374
385
|
|
|
375
386
|
if (result.hasWriteErrors()) {
|
|
376
387
|
out.errors = result.getWriteErrors();
|
|
377
|
-
for (const
|
|
378
|
-
const
|
|
379
|
-
const
|
|
380
|
-
out.counts[
|
|
388
|
+
for (const error of out.errors) {
|
|
389
|
+
const operation = operations[error.index];
|
|
390
|
+
const key = TypedObject.keys(operation)[0];
|
|
391
|
+
out.counts[key] -= 1;
|
|
381
392
|
}
|
|
382
393
|
out.counts.error = out.errors.length;
|
|
383
394
|
}
|
|
@@ -427,18 +438,18 @@ export class MongoModelService implements
|
|
|
427
438
|
|
|
428
439
|
async * listByIndex<T extends ModelType>(cls: Class<T>, idx: string, body?: DeepPartial<T>): AsyncIterable<T> {
|
|
429
440
|
const store = await this.getStore(cls);
|
|
430
|
-
const
|
|
441
|
+
const idxConfig = ModelRegistryIndex.getIndex(cls, idx, ['sorted', 'unsorted']);
|
|
431
442
|
|
|
432
443
|
const where = this.getWhereFilter(
|
|
433
444
|
cls,
|
|
434
445
|
castTo(ModelIndexedUtil.projectIndex(cls, idx, body, { emptySortValue: { $exists: true } }))
|
|
435
446
|
);
|
|
436
447
|
|
|
437
|
-
const sort = castTo<{ [ListIndexSymbol]: PlainIdx }>(
|
|
448
|
+
const sort = castTo<{ [ListIndexSymbol]: PlainIdx }>(idxConfig)[ListIndexSymbol] ??= MongoUtil.getPlainIndex(idxConfig);
|
|
438
449
|
const cursor = store.find(where, { timeout: true }).batchSize(100).sort(castTo(sort));
|
|
439
450
|
|
|
440
|
-
for await (const
|
|
441
|
-
yield await this.postLoad(cls,
|
|
451
|
+
for await (const item of cursor) {
|
|
452
|
+
yield await this.postLoad(cls, item);
|
|
442
453
|
}
|
|
443
454
|
}
|
|
444
455
|
|
|
@@ -450,7 +461,7 @@ export class MongoModelService implements
|
|
|
450
461
|
const filter = MongoUtil.extractWhereFilter(cls, query.where);
|
|
451
462
|
const cursor = col.find(filter, {});
|
|
452
463
|
const items = await MongoUtil.prepareCursor(cls, cursor, query).toArray();
|
|
453
|
-
return await Promise.all(items.map(
|
|
464
|
+
return await Promise.all(items.map(item => this.postLoad(cls, item)));
|
|
454
465
|
}
|
|
455
466
|
|
|
456
467
|
async queryCount<T extends ModelType>(cls: Class<T>, query: ModelQuery<T>): Promise<number> {
|
|
@@ -500,13 +511,13 @@ export class MongoModelService implements
|
|
|
500
511
|
const col = await this.getStore(cls);
|
|
501
512
|
const items = MongoUtil.extractSimple(cls, item);
|
|
502
513
|
const final = Object.entries(items).reduce<Partial<Record<'$unset' | '$set', Record<string, unknown>>>>(
|
|
503
|
-
(
|
|
504
|
-
if (
|
|
505
|
-
(
|
|
514
|
+
(document, [key, value]) => {
|
|
515
|
+
if (value === null || value === undefined) {
|
|
516
|
+
(document.$unset ??= {})[key] = value;
|
|
506
517
|
} else {
|
|
507
|
-
(
|
|
518
|
+
(document.$set ??= {})[key] = value;
|
|
508
519
|
}
|
|
509
|
-
return
|
|
520
|
+
return document;
|
|
510
521
|
}, {});
|
|
511
522
|
|
|
512
523
|
const filter = MongoUtil.extractWhereFilter(cls, query.where);
|
|
@@ -523,14 +534,14 @@ export class MongoModelService implements
|
|
|
523
534
|
await QueryVerifier.verify(cls, query);
|
|
524
535
|
}
|
|
525
536
|
|
|
526
|
-
let
|
|
537
|
+
let queryObject: Record<string, unknown> = { [field]: { $exists: true } };
|
|
527
538
|
|
|
528
539
|
if (query?.where) {
|
|
529
|
-
|
|
540
|
+
queryObject = { $and: [queryObject, MongoUtil.extractWhereFilter(cls, query.where)] };
|
|
530
541
|
}
|
|
531
542
|
|
|
532
543
|
const aggregations: object[] = [
|
|
533
|
-
{ $match:
|
|
544
|
+
{ $match: queryObject },
|
|
534
545
|
{
|
|
535
546
|
$group: {
|
|
536
547
|
_id: `$${field}`,
|
|
@@ -544,9 +555,9 @@ export class MongoModelService implements
|
|
|
544
555
|
const result = await col.aggregate<{ _id: ObjectId, count: number }>(aggregations).toArray();
|
|
545
556
|
|
|
546
557
|
return result
|
|
547
|
-
.map(
|
|
548
|
-
key: MongoUtil.idToString(
|
|
549
|
-
count:
|
|
558
|
+
.map(item => ({
|
|
559
|
+
key: MongoUtil.idToString(item._id),
|
|
560
|
+
count: item.count
|
|
550
561
|
}))
|
|
551
562
|
.toSorted((a, b) => b.count - a.count);
|
|
552
563
|
}
|
|
@@ -554,15 +565,15 @@ export class MongoModelService implements
|
|
|
554
565
|
// Suggest
|
|
555
566
|
async suggestValues<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<string[]> {
|
|
556
567
|
await QueryVerifier.verify(cls, query);
|
|
557
|
-
const
|
|
558
|
-
const results = await this.query<T>(cls,
|
|
568
|
+
const resolvedQuery = ModelQuerySuggestUtil.getSuggestFieldQuery<T>(cls, field, prefix, query);
|
|
569
|
+
const results = await this.query<T>(cls, resolvedQuery);
|
|
559
570
|
return ModelQuerySuggestUtil.combineSuggestResults<T, string>(cls, field, prefix, results, (a) => a, query && query.limit);
|
|
560
571
|
}
|
|
561
572
|
|
|
562
573
|
async suggest<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<T[]> {
|
|
563
574
|
await QueryVerifier.verify(cls, query);
|
|
564
|
-
const
|
|
565
|
-
const results = await this.query<T>(cls,
|
|
575
|
+
const resolvedQuery = ModelQuerySuggestUtil.getSuggestQuery<T>(cls, field, prefix, query);
|
|
576
|
+
const results = await this.query<T>(cls, resolvedQuery);
|
|
566
577
|
return ModelQuerySuggestUtil.combineSuggestResults(cls, field, prefix, results, (_, b) => b, query && query.limit);
|
|
567
578
|
}
|
|
568
579
|
|
|
@@ -582,6 +593,6 @@ export class MongoModelService implements
|
|
|
582
593
|
|
|
583
594
|
const cursor = col.find(castTo({ $and: [{ $text: search }, filter] }), {});
|
|
584
595
|
const items = await MongoUtil.prepareCursor(cls, cursor, query).toArray();
|
|
585
|
-
return await Promise.all(items.map(
|
|
596
|
+
return await Promise.all(items.map(item => this.postLoad(cls, item)));
|
|
586
597
|
}
|
|
587
598
|
}
|