@naturalcycles/datastore-lib 4.16.1 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/datastore.db.d.ts +3 -4
- package/dist/datastore.db.js +25 -36
- package/dist/datastore.model.d.ts +2 -2
- package/dist/query.util.d.ts +2 -2
- package/dist/query.util.js +3 -2
- package/package.json +2 -2
- package/src/datastore.db.ts +26 -55
- package/src/datastore.model.ts +2 -2
- package/src/query.util.ts +2 -3
package/dist/datastore.db.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Key, Transaction } from '@google-cloud/datastore';
|
|
2
|
+
import { Datastore } from '@google-cloud/datastore';
|
|
2
3
|
import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBSupport, CommonDBTransactionOptions, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
|
|
3
4
|
import { BaseCommonDB } from '@naturalcycles/db-lib';
|
|
4
5
|
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
@@ -22,9 +23,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
22
23
|
* Datastore.KEY
|
|
23
24
|
*/
|
|
24
25
|
protected KEY: symbol;
|
|
25
|
-
ds():
|
|
26
|
-
private getPropertyFilter;
|
|
27
|
-
private getDatastoreLib;
|
|
26
|
+
ds(): Datastore;
|
|
28
27
|
ping(): Promise<void>;
|
|
29
28
|
getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: DatastoreDBReadOptions): Promise<ROW[]>;
|
|
30
29
|
multiGet<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: DatastoreDBReadOptions): Promise<StringMap<ROW[]>>;
|
package/dist/datastore.db.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Datastore, PropertyFilter } from '@google-cloud/datastore';
|
|
1
2
|
import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
|
|
2
3
|
import { _round } from '@naturalcycles/js-lib';
|
|
3
4
|
import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
|
|
@@ -62,10 +63,9 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
62
63
|
*/
|
|
63
64
|
KEY;
|
|
64
65
|
// @memo() // not used to be able to connect to many DBs in the same server instance
|
|
65
|
-
|
|
66
|
+
ds() {
|
|
66
67
|
if (!this.cachedDatastore) {
|
|
67
68
|
_assert(process.env['APP_ENV'] !== 'test', 'DatastoreDB cannot be used in Test env, please use InMemoryDB');
|
|
68
|
-
const DS = (await this.getDatastoreLib()).Datastore;
|
|
69
69
|
this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT'];
|
|
70
70
|
if (this.cfg.projectId) {
|
|
71
71
|
this.cfg.logger.log(`DatastoreDB connected to ${boldWhite(this.cfg.projectId)}`);
|
|
@@ -76,26 +76,18 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
76
76
|
if (this.cfg.grpc) {
|
|
77
77
|
this.cfg.logger.log('!!! DatastoreDB using custom grpc !!!');
|
|
78
78
|
}
|
|
79
|
-
this.cachedDatastore = new
|
|
79
|
+
this.cachedDatastore = new Datastore(this.cfg);
|
|
80
80
|
this.KEY = this.cachedDatastore.KEY;
|
|
81
81
|
}
|
|
82
82
|
return this.cachedDatastore;
|
|
83
83
|
}
|
|
84
|
-
async getPropertyFilter() {
|
|
85
|
-
return (await this.getDatastoreLib()).PropertyFilter;
|
|
86
|
-
}
|
|
87
|
-
async getDatastoreLib() {
|
|
88
|
-
// Lazy-loading
|
|
89
|
-
const lib = await import('@google-cloud/datastore');
|
|
90
|
-
return lib;
|
|
91
|
-
}
|
|
92
84
|
async ping() {
|
|
93
85
|
await this.getAllStats();
|
|
94
86
|
}
|
|
95
87
|
async getByIds(table, ids, opt = {}) {
|
|
96
88
|
if (!ids.length)
|
|
97
89
|
return [];
|
|
98
|
-
let ds =
|
|
90
|
+
let ds = this.ds();
|
|
99
91
|
const keys = ids.map(id => this.key(ds, table, id));
|
|
100
92
|
let rows;
|
|
101
93
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
@@ -115,9 +107,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
115
107
|
}
|
|
116
108
|
this.cfg.logger.log(`datastore recreated on timeout (${_ms(this.cfg.timeout)}) while loading ${table}`);
|
|
117
109
|
// This is to debug "GCP Datastore Timeout issue"
|
|
118
|
-
|
|
119
|
-
const DS = datastoreLib.Datastore;
|
|
120
|
-
ds = this.cachedDatastore = new DS(this.cfg);
|
|
110
|
+
ds = this.cachedDatastore = new Datastore(this.cfg);
|
|
121
111
|
// Second try (will throw)
|
|
122
112
|
try {
|
|
123
113
|
const r = await pRetry(() => (opt.tx?.tx || ds).get(keys, dsOpt), {
|
|
@@ -150,7 +140,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
150
140
|
}
|
|
151
141
|
async multiGet(map, opt = {}) {
|
|
152
142
|
const result = {};
|
|
153
|
-
const ds =
|
|
143
|
+
const ds = this.ds();
|
|
154
144
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
155
145
|
const keys = [];
|
|
156
146
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
@@ -182,8 +172,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
182
172
|
rows: await this.getByIds(dbQuery.table, ids, opt),
|
|
183
173
|
};
|
|
184
174
|
}
|
|
185
|
-
const ds =
|
|
186
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
175
|
+
const ds = this.ds();
|
|
176
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
187
177
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
188
178
|
const qr = await this.runDatastoreQuery(q, dsOpt);
|
|
189
179
|
// Special case when projection query didn't specify 'id'
|
|
@@ -193,15 +183,15 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
193
183
|
return qr;
|
|
194
184
|
}
|
|
195
185
|
async runQueryCount(dbQuery, opt = {}) {
|
|
196
|
-
const ds =
|
|
197
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
186
|
+
const ds = this.ds();
|
|
187
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
198
188
|
const aq = ds.createAggregationQuery(q).count('count');
|
|
199
189
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
200
190
|
const [entities] = await ds.runAggregationQuery(aq, dsOpt);
|
|
201
191
|
return entities[0]?.count;
|
|
202
192
|
}
|
|
203
193
|
async runDatastoreQuery(q, dsOpt) {
|
|
204
|
-
const ds =
|
|
194
|
+
const ds = this.ds();
|
|
205
195
|
const [entities, queryResult] = await ds.runQuery(q, dsOpt);
|
|
206
196
|
const rows = entities.map(e => this.mapId(e));
|
|
207
197
|
return {
|
|
@@ -211,8 +201,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
211
201
|
}
|
|
212
202
|
streamQuery(dbQuery, _opt) {
|
|
213
203
|
return Pipeline.fromAsyncReadable(async () => {
|
|
214
|
-
const ds =
|
|
215
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
204
|
+
const ds = this.ds();
|
|
205
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
216
206
|
const opt = {
|
|
217
207
|
logger: this.cfg.logger,
|
|
218
208
|
...this.cfg.streamOptions,
|
|
@@ -228,7 +218,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
228
218
|
* Returns saved entities with generated id/updated/created (non-mutating!)
|
|
229
219
|
*/
|
|
230
220
|
async saveBatch(table, rows, opt = {}) {
|
|
231
|
-
const ds =
|
|
221
|
+
const ds = this.ds();
|
|
232
222
|
const entities = rows.map(obj => this.toDatastoreEntity(ds, table, obj, opt.excludeFromIndexes));
|
|
233
223
|
const method = methodMap[opt.saveMethod || 'upsert'] || 'save';
|
|
234
224
|
const save = pRetryFn(async (batch) => {
|
|
@@ -264,8 +254,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
264
254
|
const ids = idFilter.op === '==' ? [idFilter.val] : idFilter.val;
|
|
265
255
|
return await this.deleteByIds(q.table, ids, opt);
|
|
266
256
|
}
|
|
267
|
-
const ds =
|
|
268
|
-
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table)
|
|
257
|
+
const ds = this.ds();
|
|
258
|
+
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table));
|
|
269
259
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
270
260
|
const { rows } = await this.runDatastoreQuery(datastoreQuery, dsOpt);
|
|
271
261
|
return await this.deleteByIds(q.table, rows.map(obj => obj.id), opt);
|
|
@@ -275,7 +265,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
275
265
|
* regardless if they were actually deleted or not.
|
|
276
266
|
*/
|
|
277
267
|
async deleteByIds(table, ids, opt = {}) {
|
|
278
|
-
const ds =
|
|
268
|
+
const ds = this.ds();
|
|
279
269
|
const keys = ids.map(id => this.key(ds, table, id));
|
|
280
270
|
const retryOptions = this.getPRetryOptions(`DatastoreLib.deleteByIds(${table})`);
|
|
281
271
|
await pMap(_chunk(keys, MAX_ITEMS),
|
|
@@ -290,7 +280,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
290
280
|
return ids.length;
|
|
291
281
|
}
|
|
292
282
|
async multiDelete(map, opt = {}) {
|
|
293
|
-
const ds =
|
|
283
|
+
const ds = this.ds();
|
|
294
284
|
const keys = [];
|
|
295
285
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
296
286
|
keys.push(...ids.map(id => this.key(ds, table, id)));
|
|
@@ -308,7 +298,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
308
298
|
return keys.length;
|
|
309
299
|
}
|
|
310
300
|
async createTransaction(opt = {}) {
|
|
311
|
-
const ds =
|
|
301
|
+
const ds = this.ds();
|
|
312
302
|
const { readOnly } = opt;
|
|
313
303
|
const datastoreTx = ds.transaction({
|
|
314
304
|
readOnly,
|
|
@@ -317,7 +307,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
317
307
|
return new DatastoreDBTransaction(this, datastoreTx);
|
|
318
308
|
}
|
|
319
309
|
async runInTransaction(fn, opt = {}) {
|
|
320
|
-
const ds =
|
|
310
|
+
const ds = this.ds();
|
|
321
311
|
const { readOnly } = opt;
|
|
322
312
|
const datastoreTx = ds.transaction({
|
|
323
313
|
readOnly,
|
|
@@ -334,7 +324,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
334
324
|
}
|
|
335
325
|
}
|
|
336
326
|
async getAllStats() {
|
|
337
|
-
const ds =
|
|
327
|
+
const ds = this.ds();
|
|
338
328
|
const q = ds.createQuery('__Stat_Kind__');
|
|
339
329
|
const [statsArray] = await ds.runQuery(q);
|
|
340
330
|
return statsArray || [];
|
|
@@ -343,12 +333,11 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
343
333
|
* Returns undefined e.g when Table is non-existing
|
|
344
334
|
*/
|
|
345
335
|
async getStats(table) {
|
|
346
|
-
const ds =
|
|
347
|
-
const propertyFilter = await this.getPropertyFilter();
|
|
336
|
+
const ds = this.ds();
|
|
348
337
|
const q = ds
|
|
349
338
|
.createQuery('__Stat_Kind__')
|
|
350
339
|
// .filter('kind_name', table)
|
|
351
|
-
.filter(new
|
|
340
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
352
341
|
.limit(1);
|
|
353
342
|
const [statsArray] = await ds.runQuery(q);
|
|
354
343
|
const [stats] = statsArray;
|
|
@@ -359,11 +348,11 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
359
348
|
return stats?.count;
|
|
360
349
|
}
|
|
361
350
|
async getTableProperties(table) {
|
|
362
|
-
const ds =
|
|
351
|
+
const ds = this.ds();
|
|
363
352
|
const q = ds
|
|
364
353
|
.createQuery('__Stat_PropertyType_PropertyName_Kind__')
|
|
365
354
|
// .filter('kind_name', table)
|
|
366
|
-
.filter(new (
|
|
355
|
+
.filter(new PropertyFilter('kind_name', '=', table));
|
|
367
356
|
const [stats] = await ds.runQuery(q);
|
|
368
357
|
return stats;
|
|
369
358
|
}
|
|
@@ -106,12 +106,12 @@ export interface DatastoreStats {
|
|
|
106
106
|
export declare enum DatastoreType {
|
|
107
107
|
Blob = "Blob",
|
|
108
108
|
Text = "Text",
|
|
109
|
-
String = "String",// eslint-disable-line id-
|
|
109
|
+
String = "String",// eslint-disable-line id-denylist
|
|
110
110
|
EmbeddedEntity = "EmbeddedEntity",
|
|
111
111
|
Float = "Float",
|
|
112
112
|
Integer = "Integer",
|
|
113
113
|
DATE_TIME = "Date/Time",
|
|
114
|
-
Boolean = "Boolean",// eslint-disable-line id-
|
|
114
|
+
Boolean = "Boolean",// eslint-disable-line id-denylist
|
|
115
115
|
NULL = "NULL"
|
|
116
116
|
}
|
|
117
117
|
export interface DatastorePropertyStats {
|
package/dist/query.util.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type Query } from '@google-cloud/datastore';
|
|
2
2
|
import type { DBQuery } from '@naturalcycles/db-lib';
|
|
3
3
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
4
|
-
export declare function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query
|
|
4
|
+
export declare function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query): Query;
|
package/dist/query.util.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { PropertyFilter } from '@google-cloud/datastore';
|
|
1
2
|
const FNAME_MAP = {
|
|
2
3
|
id: '__key__',
|
|
3
4
|
};
|
|
@@ -6,7 +7,7 @@ const OP_MAP = {
|
|
|
6
7
|
in: 'IN',
|
|
7
8
|
'not-in': 'NOT_IN',
|
|
8
9
|
};
|
|
9
|
-
export function dbQueryToDatastoreQuery(dbQuery, emptyQuery
|
|
10
|
+
export function dbQueryToDatastoreQuery(dbQuery, emptyQuery) {
|
|
10
11
|
let q = emptyQuery;
|
|
11
12
|
// filter
|
|
12
13
|
for (const f of dbQuery._filters) {
|
|
@@ -20,7 +21,7 @@ export function dbQueryToDatastoreQuery(dbQuery, emptyQuery, propertyFilterClass
|
|
|
20
21
|
let { op, val } = f;
|
|
21
22
|
if (val === undefined)
|
|
22
23
|
val = null;
|
|
23
|
-
q = q.filter(new
|
|
24
|
+
q = q.filter(new PropertyFilter(f.name, OP_MAP[op] || op, val));
|
|
24
25
|
}
|
|
25
26
|
// limit
|
|
26
27
|
q = q.limit(dbQuery._limitValue || 0);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/datastore-lib",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "4.
|
|
4
|
+
"version": "4.17.0",
|
|
5
5
|
"description": "Opinionated library to work with Google Datastore, implements CommonDB",
|
|
6
6
|
"dependencies": {
|
|
7
7
|
"@google-cloud/datastore": "^10",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
},
|
|
12
12
|
"devDependencies": {
|
|
13
13
|
"@types/node": "^24",
|
|
14
|
-
"@naturalcycles/dev-lib": "
|
|
14
|
+
"@naturalcycles/dev-lib": "20.6.0"
|
|
15
15
|
},
|
|
16
16
|
"exports": {
|
|
17
17
|
".": "./dist/index.js"
|
package/src/datastore.db.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Key, Query, Transaction } from '@google-cloud/datastore'
|
|
2
|
+
import { Datastore, PropertyFilter } from '@google-cloud/datastore'
|
|
2
3
|
import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
|
|
3
4
|
import type {
|
|
4
5
|
CommonDB,
|
|
@@ -113,14 +114,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
113
114
|
protected KEY!: symbol
|
|
114
115
|
|
|
115
116
|
// @memo() // not used to be able to connect to many DBs in the same server instance
|
|
116
|
-
|
|
117
|
+
ds(): Datastore {
|
|
117
118
|
if (!this.cachedDatastore) {
|
|
118
119
|
_assert(
|
|
119
120
|
process.env['APP_ENV'] !== 'test',
|
|
120
121
|
'DatastoreDB cannot be used in Test env, please use InMemoryDB',
|
|
121
122
|
)
|
|
122
123
|
|
|
123
|
-
const DS = (await this.getDatastoreLib()).Datastore as typeof Datastore
|
|
124
124
|
this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT']
|
|
125
125
|
|
|
126
126
|
if (this.cfg.projectId) {
|
|
@@ -133,23 +133,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
133
133
|
this.cfg.logger.log('!!! DatastoreDB using custom grpc !!!')
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
-
this.cachedDatastore = new
|
|
136
|
+
this.cachedDatastore = new Datastore(this.cfg)
|
|
137
137
|
this.KEY = this.cachedDatastore.KEY
|
|
138
138
|
}
|
|
139
139
|
|
|
140
140
|
return this.cachedDatastore
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
-
private async getPropertyFilter(): Promise<typeof PropertyFilter> {
|
|
144
|
-
return (await this.getDatastoreLib()).PropertyFilter
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
private async getDatastoreLib(): Promise<any> {
|
|
148
|
-
// Lazy-loading
|
|
149
|
-
const lib = await import('@google-cloud/datastore')
|
|
150
|
-
return lib
|
|
151
|
-
}
|
|
152
|
-
|
|
153
143
|
override async ping(): Promise<void> {
|
|
154
144
|
await this.getAllStats()
|
|
155
145
|
}
|
|
@@ -160,7 +150,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
160
150
|
opt: DatastoreDBReadOptions = {},
|
|
161
151
|
): Promise<ROW[]> {
|
|
162
152
|
if (!ids.length) return []
|
|
163
|
-
let ds =
|
|
153
|
+
let ds = this.ds()
|
|
164
154
|
const keys = ids.map(id => this.key(ds, table, id))
|
|
165
155
|
let rows: any[]
|
|
166
156
|
|
|
@@ -188,9 +178,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
188
178
|
)
|
|
189
179
|
|
|
190
180
|
// This is to debug "GCP Datastore Timeout issue"
|
|
191
|
-
|
|
192
|
-
const DS = datastoreLib.Datastore as typeof Datastore
|
|
193
|
-
ds = this.cachedDatastore = new DS(this.cfg)
|
|
181
|
+
ds = this.cachedDatastore = new Datastore(this.cfg)
|
|
194
182
|
|
|
195
183
|
// Second try (will throw)
|
|
196
184
|
try {
|
|
@@ -235,7 +223,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
235
223
|
opt: DatastoreDBReadOptions = {},
|
|
236
224
|
): Promise<StringMap<ROW[]>> {
|
|
237
225
|
const result: StringMap<ROW[]> = {}
|
|
238
|
-
const ds =
|
|
226
|
+
const ds = this.ds()
|
|
239
227
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
240
228
|
const keys: Key[] = []
|
|
241
229
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
@@ -278,12 +266,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
278
266
|
}
|
|
279
267
|
}
|
|
280
268
|
|
|
281
|
-
const ds =
|
|
282
|
-
const q = dbQueryToDatastoreQuery(
|
|
283
|
-
dbQuery,
|
|
284
|
-
ds.createQuery(dbQuery.table),
|
|
285
|
-
await this.getPropertyFilter(),
|
|
286
|
-
)
|
|
269
|
+
const ds = this.ds()
|
|
270
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
287
271
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
288
272
|
const qr = await this.runDatastoreQuery<ROW>(q, dsOpt)
|
|
289
273
|
|
|
@@ -299,12 +283,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
299
283
|
dbQuery: DBQuery<ROW>,
|
|
300
284
|
opt: DatastoreDBReadOptions = {},
|
|
301
285
|
): Promise<number> {
|
|
302
|
-
const ds =
|
|
303
|
-
const q = dbQueryToDatastoreQuery(
|
|
304
|
-
dbQuery,
|
|
305
|
-
ds.createQuery(dbQuery.table),
|
|
306
|
-
await this.getPropertyFilter(),
|
|
307
|
-
)
|
|
286
|
+
const ds = this.ds()
|
|
287
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
308
288
|
const aq = ds.createAggregationQuery(q).count('count')
|
|
309
289
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
310
290
|
const [entities] = await ds.runAggregationQuery(aq, dsOpt)
|
|
@@ -315,7 +295,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
315
295
|
q: Query,
|
|
316
296
|
dsOpt: RunQueryOptions,
|
|
317
297
|
): Promise<RunQueryResult<ROW>> {
|
|
318
|
-
const ds =
|
|
298
|
+
const ds = this.ds()
|
|
319
299
|
const [entities, queryResult] = await ds.runQuery(q, dsOpt)
|
|
320
300
|
|
|
321
301
|
const rows = entities.map(e => this.mapId<ROW>(e))
|
|
@@ -331,13 +311,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
331
311
|
_opt?: DatastoreDBStreamOptions,
|
|
332
312
|
): Pipeline<ROW> {
|
|
333
313
|
return Pipeline.fromAsyncReadable<ROW>(async () => {
|
|
334
|
-
const ds =
|
|
314
|
+
const ds = this.ds()
|
|
335
315
|
|
|
336
|
-
const q = dbQueryToDatastoreQuery(
|
|
337
|
-
dbQuery,
|
|
338
|
-
ds.createQuery(dbQuery.table),
|
|
339
|
-
await this.getPropertyFilter(),
|
|
340
|
-
)
|
|
316
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
341
317
|
|
|
342
318
|
const opt = {
|
|
343
319
|
logger: this.cfg.logger,
|
|
@@ -361,7 +337,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
361
337
|
rows: ROW[],
|
|
362
338
|
opt: DatastoreDBSaveOptions<ROW> = {},
|
|
363
339
|
): Promise<void> {
|
|
364
|
-
const ds =
|
|
340
|
+
const ds = this.ds()
|
|
365
341
|
const entities = rows.map(obj =>
|
|
366
342
|
this.toDatastoreEntity(ds, table, obj, opt.excludeFromIndexes as string[]),
|
|
367
343
|
)
|
|
@@ -414,12 +390,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
414
390
|
return await this.deleteByIds(q.table, ids, opt)
|
|
415
391
|
}
|
|
416
392
|
|
|
417
|
-
const ds =
|
|
418
|
-
const datastoreQuery = dbQueryToDatastoreQuery(
|
|
419
|
-
q.select([]),
|
|
420
|
-
ds.createQuery(q.table),
|
|
421
|
-
await this.getPropertyFilter(),
|
|
422
|
-
)
|
|
393
|
+
const ds = this.ds()
|
|
394
|
+
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table))
|
|
423
395
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
424
396
|
const { rows } = await this.runDatastoreQuery<ObjectWithId>(datastoreQuery, dsOpt)
|
|
425
397
|
return await this.deleteByIds(
|
|
@@ -438,7 +410,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
438
410
|
ids: string[],
|
|
439
411
|
opt: DatastoreDBOptions = {},
|
|
440
412
|
): Promise<number> {
|
|
441
|
-
const ds =
|
|
413
|
+
const ds = this.ds()
|
|
442
414
|
const keys = ids.map(id => this.key(ds, table, id))
|
|
443
415
|
|
|
444
416
|
const retryOptions = this.getPRetryOptions(`DatastoreLib.deleteByIds(${table})`)
|
|
@@ -462,7 +434,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
462
434
|
map: StringMap<string[]>,
|
|
463
435
|
opt: DatastoreDBOptions = {},
|
|
464
436
|
): Promise<number> {
|
|
465
|
-
const ds =
|
|
437
|
+
const ds = this.ds()
|
|
466
438
|
const keys: Key[] = []
|
|
467
439
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
468
440
|
keys.push(...ids.map(id => this.key(ds, table, id)))
|
|
@@ -489,7 +461,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
489
461
|
override async createTransaction(
|
|
490
462
|
opt: CommonDBTransactionOptions = {},
|
|
491
463
|
): Promise<DatastoreDBTransaction> {
|
|
492
|
-
const ds =
|
|
464
|
+
const ds = this.ds()
|
|
493
465
|
const { readOnly } = opt
|
|
494
466
|
const datastoreTx = ds.transaction({
|
|
495
467
|
readOnly,
|
|
@@ -502,7 +474,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
502
474
|
fn: DBTransactionFn,
|
|
503
475
|
opt: CommonDBTransactionOptions = {},
|
|
504
476
|
): Promise<void> {
|
|
505
|
-
const ds =
|
|
477
|
+
const ds = this.ds()
|
|
506
478
|
const { readOnly } = opt
|
|
507
479
|
const datastoreTx = ds.transaction({
|
|
508
480
|
readOnly,
|
|
@@ -520,7 +492,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
520
492
|
}
|
|
521
493
|
|
|
522
494
|
async getAllStats(): Promise<DatastoreStats[]> {
|
|
523
|
-
const ds =
|
|
495
|
+
const ds = this.ds()
|
|
524
496
|
const q = ds.createQuery('__Stat_Kind__')
|
|
525
497
|
const [statsArray] = await ds.runQuery(q)
|
|
526
498
|
return statsArray || []
|
|
@@ -530,13 +502,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
530
502
|
* Returns undefined e.g when Table is non-existing
|
|
531
503
|
*/
|
|
532
504
|
async getStats(table: string): Promise<DatastoreStats | undefined> {
|
|
533
|
-
const ds =
|
|
534
|
-
const propertyFilter = await this.getPropertyFilter()
|
|
505
|
+
const ds = this.ds()
|
|
535
506
|
|
|
536
507
|
const q = ds
|
|
537
508
|
.createQuery('__Stat_Kind__')
|
|
538
509
|
// .filter('kind_name', table)
|
|
539
|
-
.filter(new
|
|
510
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
540
511
|
.limit(1)
|
|
541
512
|
const [statsArray] = await ds.runQuery(q)
|
|
542
513
|
const [stats] = statsArray
|
|
@@ -549,11 +520,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
549
520
|
}
|
|
550
521
|
|
|
551
522
|
async getTableProperties(table: string): Promise<DatastorePropertyStats[]> {
|
|
552
|
-
const ds =
|
|
523
|
+
const ds = this.ds()
|
|
553
524
|
const q = ds
|
|
554
525
|
.createQuery('__Stat_PropertyType_PropertyName_Kind__')
|
|
555
526
|
// .filter('kind_name', table)
|
|
556
|
-
.filter(new (
|
|
527
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
557
528
|
const [stats] = await ds.runQuery(q)
|
|
558
529
|
return stats
|
|
559
530
|
}
|
package/src/datastore.model.ts
CHANGED
|
@@ -128,12 +128,12 @@ export interface DatastoreStats {
|
|
|
128
128
|
export enum DatastoreType {
|
|
129
129
|
Blob = 'Blob',
|
|
130
130
|
Text = 'Text',
|
|
131
|
-
String = 'String', // eslint-disable-line id-
|
|
131
|
+
String = 'String', // eslint-disable-line id-denylist
|
|
132
132
|
EmbeddedEntity = 'EmbeddedEntity',
|
|
133
133
|
Float = 'Float',
|
|
134
134
|
Integer = 'Integer',
|
|
135
135
|
DATE_TIME = 'Date/Time',
|
|
136
|
-
Boolean = 'Boolean', // eslint-disable-line id-
|
|
136
|
+
Boolean = 'Boolean', // eslint-disable-line id-denylist
|
|
137
137
|
NULL = 'NULL',
|
|
138
138
|
}
|
|
139
139
|
|
package/src/query.util.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { PropertyFilter, type Query } from '@google-cloud/datastore'
|
|
2
2
|
import type { DBQuery, DBQueryFilterOperator } from '@naturalcycles/db-lib'
|
|
3
3
|
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
|
|
4
4
|
|
|
@@ -15,7 +15,6 @@ const OP_MAP: Partial<Record<DBQueryFilterOperator, string>> = {
|
|
|
15
15
|
export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
|
|
16
16
|
dbQuery: Readonly<DBQuery<ROW>>,
|
|
17
17
|
emptyQuery: Query,
|
|
18
|
-
propertyFilterClass: typeof PropertyFilter,
|
|
19
18
|
): Query {
|
|
20
19
|
let q = emptyQuery
|
|
21
20
|
|
|
@@ -31,7 +30,7 @@ export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
|
|
|
31
30
|
// `a == null` will return just that - rows with null values
|
|
32
31
|
let { op, val } = f
|
|
33
32
|
if (val === undefined) val = null
|
|
34
|
-
q = q.filter(new
|
|
33
|
+
q = q.filter(new PropertyFilter(f.name as string, OP_MAP[op] || (op as any), val))
|
|
35
34
|
}
|
|
36
35
|
|
|
37
36
|
// limit
|