@naturalcycles/datastore-lib 4.16.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/datastore.db.d.ts +3 -4
- package/dist/datastore.db.js +27 -37
- package/dist/datastore.model.d.ts +2 -2
- package/dist/query.util.d.ts +2 -2
- package/dist/query.util.js +3 -2
- package/package.json +2 -2
- package/src/datastore.db.ts +30 -56
- package/src/datastore.model.ts +2 -2
- package/src/query.util.ts +2 -3
package/dist/datastore.db.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Key, Transaction } from '@google-cloud/datastore';
|
|
2
|
+
import { Datastore } from '@google-cloud/datastore';
|
|
2
3
|
import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBSupport, CommonDBTransactionOptions, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
|
|
3
4
|
import { BaseCommonDB } from '@naturalcycles/db-lib';
|
|
4
5
|
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
@@ -22,9 +23,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
22
23
|
* Datastore.KEY
|
|
23
24
|
*/
|
|
24
25
|
protected KEY: symbol;
|
|
25
|
-
ds():
|
|
26
|
-
private getPropertyFilter;
|
|
27
|
-
private getDatastoreLib;
|
|
26
|
+
ds(): Datastore;
|
|
28
27
|
ping(): Promise<void>;
|
|
29
28
|
getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: DatastoreDBReadOptions): Promise<ROW[]>;
|
|
30
29
|
multiGet<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: DatastoreDBReadOptions): Promise<StringMap<ROW[]>>;
|
package/dist/datastore.db.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { Datastore, PropertyFilter } from '@google-cloud/datastore';
|
|
1
2
|
import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
|
|
2
3
|
import { _round } from '@naturalcycles/js-lib';
|
|
3
4
|
import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
|
|
5
|
+
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
|
|
4
6
|
import { _assert } from '@naturalcycles/js-lib/error/assert.js';
|
|
5
7
|
import { _errorDataAppend, TimeoutError } from '@naturalcycles/js-lib/error/error.util.js';
|
|
6
8
|
import { _omit } from '@naturalcycles/js-lib/object/object.util.js';
|
|
@@ -61,10 +63,9 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
61
63
|
*/
|
|
62
64
|
KEY;
|
|
63
65
|
// @memo() // not used to be able to connect to many DBs in the same server instance
|
|
64
|
-
|
|
66
|
+
ds() {
|
|
65
67
|
if (!this.cachedDatastore) {
|
|
66
68
|
_assert(process.env['APP_ENV'] !== 'test', 'DatastoreDB cannot be used in Test env, please use InMemoryDB');
|
|
67
|
-
const DS = (await this.getDatastoreLib()).Datastore;
|
|
68
69
|
this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT'];
|
|
69
70
|
if (this.cfg.projectId) {
|
|
70
71
|
this.cfg.logger.log(`DatastoreDB connected to ${boldWhite(this.cfg.projectId)}`);
|
|
@@ -75,26 +76,18 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
75
76
|
if (this.cfg.grpc) {
|
|
76
77
|
this.cfg.logger.log('!!! DatastoreDB using custom grpc !!!');
|
|
77
78
|
}
|
|
78
|
-
this.cachedDatastore = new
|
|
79
|
+
this.cachedDatastore = new Datastore(this.cfg);
|
|
79
80
|
this.KEY = this.cachedDatastore.KEY;
|
|
80
81
|
}
|
|
81
82
|
return this.cachedDatastore;
|
|
82
83
|
}
|
|
83
|
-
async getPropertyFilter() {
|
|
84
|
-
return (await this.getDatastoreLib()).PropertyFilter;
|
|
85
|
-
}
|
|
86
|
-
async getDatastoreLib() {
|
|
87
|
-
// Lazy-loading
|
|
88
|
-
const lib = await import('@google-cloud/datastore');
|
|
89
|
-
return lib;
|
|
90
|
-
}
|
|
91
84
|
async ping() {
|
|
92
85
|
await this.getAllStats();
|
|
93
86
|
}
|
|
94
87
|
async getByIds(table, ids, opt = {}) {
|
|
95
88
|
if (!ids.length)
|
|
96
89
|
return [];
|
|
97
|
-
let ds =
|
|
90
|
+
let ds = this.ds();
|
|
98
91
|
const keys = ids.map(id => this.key(ds, table, id));
|
|
99
92
|
let rows;
|
|
100
93
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
@@ -112,11 +105,9 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
112
105
|
// Not a timeout error, re-throw
|
|
113
106
|
throw err;
|
|
114
107
|
}
|
|
115
|
-
this.cfg.logger.log(
|
|
108
|
+
this.cfg.logger.log(`datastore recreated on timeout (${_ms(this.cfg.timeout)}) while loading ${table}`);
|
|
116
109
|
// This is to debug "GCP Datastore Timeout issue"
|
|
117
|
-
|
|
118
|
-
const DS = datastoreLib.Datastore;
|
|
119
|
-
ds = this.cachedDatastore = new DS(this.cfg);
|
|
110
|
+
ds = this.cachedDatastore = new Datastore(this.cfg);
|
|
120
111
|
// Second try (will throw)
|
|
121
112
|
try {
|
|
122
113
|
const r = await pRetry(() => (opt.tx?.tx || ds).get(keys, dsOpt), {
|
|
@@ -149,7 +140,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
149
140
|
}
|
|
150
141
|
async multiGet(map, opt = {}) {
|
|
151
142
|
const result = {};
|
|
152
|
-
const ds =
|
|
143
|
+
const ds = this.ds();
|
|
153
144
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
154
145
|
const keys = [];
|
|
155
146
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
@@ -181,8 +172,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
181
172
|
rows: await this.getByIds(dbQuery.table, ids, opt),
|
|
182
173
|
};
|
|
183
174
|
}
|
|
184
|
-
const ds =
|
|
185
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
175
|
+
const ds = this.ds();
|
|
176
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
186
177
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
187
178
|
const qr = await this.runDatastoreQuery(q, dsOpt);
|
|
188
179
|
// Special case when projection query didn't specify 'id'
|
|
@@ -192,15 +183,15 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
192
183
|
return qr;
|
|
193
184
|
}
|
|
194
185
|
async runQueryCount(dbQuery, opt = {}) {
|
|
195
|
-
const ds =
|
|
196
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
186
|
+
const ds = this.ds();
|
|
187
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
197
188
|
const aq = ds.createAggregationQuery(q).count('count');
|
|
198
189
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
199
190
|
const [entities] = await ds.runAggregationQuery(aq, dsOpt);
|
|
200
191
|
return entities[0]?.count;
|
|
201
192
|
}
|
|
202
193
|
async runDatastoreQuery(q, dsOpt) {
|
|
203
|
-
const ds =
|
|
194
|
+
const ds = this.ds();
|
|
204
195
|
const [entities, queryResult] = await ds.runQuery(q, dsOpt);
|
|
205
196
|
const rows = entities.map(e => this.mapId(e));
|
|
206
197
|
return {
|
|
@@ -210,8 +201,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
210
201
|
}
|
|
211
202
|
streamQuery(dbQuery, _opt) {
|
|
212
203
|
return Pipeline.fromAsyncReadable(async () => {
|
|
213
|
-
const ds =
|
|
214
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
204
|
+
const ds = this.ds();
|
|
205
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
215
206
|
const opt = {
|
|
216
207
|
logger: this.cfg.logger,
|
|
217
208
|
...this.cfg.streamOptions,
|
|
@@ -227,7 +218,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
227
218
|
* Returns saved entities with generated id/updated/created (non-mutating!)
|
|
228
219
|
*/
|
|
229
220
|
async saveBatch(table, rows, opt = {}) {
|
|
230
|
-
const ds =
|
|
221
|
+
const ds = this.ds();
|
|
231
222
|
const entities = rows.map(obj => this.toDatastoreEntity(ds, table, obj, opt.excludeFromIndexes));
|
|
232
223
|
const method = methodMap[opt.saveMethod || 'upsert'] || 'save';
|
|
233
224
|
const save = pRetryFn(async (batch) => {
|
|
@@ -263,8 +254,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
263
254
|
const ids = idFilter.op === '==' ? [idFilter.val] : idFilter.val;
|
|
264
255
|
return await this.deleteByIds(q.table, ids, opt);
|
|
265
256
|
}
|
|
266
|
-
const ds =
|
|
267
|
-
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table)
|
|
257
|
+
const ds = this.ds();
|
|
258
|
+
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table));
|
|
268
259
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
269
260
|
const { rows } = await this.runDatastoreQuery(datastoreQuery, dsOpt);
|
|
270
261
|
return await this.deleteByIds(q.table, rows.map(obj => obj.id), opt);
|
|
@@ -274,7 +265,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
274
265
|
* regardless if they were actually deleted or not.
|
|
275
266
|
*/
|
|
276
267
|
async deleteByIds(table, ids, opt = {}) {
|
|
277
|
-
const ds =
|
|
268
|
+
const ds = this.ds();
|
|
278
269
|
const keys = ids.map(id => this.key(ds, table, id));
|
|
279
270
|
const retryOptions = this.getPRetryOptions(`DatastoreLib.deleteByIds(${table})`);
|
|
280
271
|
await pMap(_chunk(keys, MAX_ITEMS),
|
|
@@ -289,7 +280,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
289
280
|
return ids.length;
|
|
290
281
|
}
|
|
291
282
|
async multiDelete(map, opt = {}) {
|
|
292
|
-
const ds =
|
|
283
|
+
const ds = this.ds();
|
|
293
284
|
const keys = [];
|
|
294
285
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
295
286
|
keys.push(...ids.map(id => this.key(ds, table, id)));
|
|
@@ -307,7 +298,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
307
298
|
return keys.length;
|
|
308
299
|
}
|
|
309
300
|
async createTransaction(opt = {}) {
|
|
310
|
-
const ds =
|
|
301
|
+
const ds = this.ds();
|
|
311
302
|
const { readOnly } = opt;
|
|
312
303
|
const datastoreTx = ds.transaction({
|
|
313
304
|
readOnly,
|
|
@@ -316,7 +307,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
316
307
|
return new DatastoreDBTransaction(this, datastoreTx);
|
|
317
308
|
}
|
|
318
309
|
async runInTransaction(fn, opt = {}) {
|
|
319
|
-
const ds =
|
|
310
|
+
const ds = this.ds();
|
|
320
311
|
const { readOnly } = opt;
|
|
321
312
|
const datastoreTx = ds.transaction({
|
|
322
313
|
readOnly,
|
|
@@ -333,7 +324,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
333
324
|
}
|
|
334
325
|
}
|
|
335
326
|
async getAllStats() {
|
|
336
|
-
const ds =
|
|
327
|
+
const ds = this.ds();
|
|
337
328
|
const q = ds.createQuery('__Stat_Kind__');
|
|
338
329
|
const [statsArray] = await ds.runQuery(q);
|
|
339
330
|
return statsArray || [];
|
|
@@ -342,12 +333,11 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
342
333
|
* Returns undefined e.g when Table is non-existing
|
|
343
334
|
*/
|
|
344
335
|
async getStats(table) {
|
|
345
|
-
const ds =
|
|
346
|
-
const propertyFilter = await this.getPropertyFilter();
|
|
336
|
+
const ds = this.ds();
|
|
347
337
|
const q = ds
|
|
348
338
|
.createQuery('__Stat_Kind__')
|
|
349
339
|
// .filter('kind_name', table)
|
|
350
|
-
.filter(new
|
|
340
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
351
341
|
.limit(1);
|
|
352
342
|
const [statsArray] = await ds.runQuery(q);
|
|
353
343
|
const [stats] = statsArray;
|
|
@@ -358,11 +348,11 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
358
348
|
return stats?.count;
|
|
359
349
|
}
|
|
360
350
|
async getTableProperties(table) {
|
|
361
|
-
const ds =
|
|
351
|
+
const ds = this.ds();
|
|
362
352
|
const q = ds
|
|
363
353
|
.createQuery('__Stat_PropertyType_PropertyName_Kind__')
|
|
364
354
|
// .filter('kind_name', table)
|
|
365
|
-
.filter(new (
|
|
355
|
+
.filter(new PropertyFilter('kind_name', '=', table));
|
|
366
356
|
const [stats] = await ds.runQuery(q);
|
|
367
357
|
return stats;
|
|
368
358
|
}
|
|
@@ -106,12 +106,12 @@ export interface DatastoreStats {
|
|
|
106
106
|
export declare enum DatastoreType {
|
|
107
107
|
Blob = "Blob",
|
|
108
108
|
Text = "Text",
|
|
109
|
-
String = "String",// eslint-disable-line id-
|
|
109
|
+
String = "String",// eslint-disable-line id-denylist
|
|
110
110
|
EmbeddedEntity = "EmbeddedEntity",
|
|
111
111
|
Float = "Float",
|
|
112
112
|
Integer = "Integer",
|
|
113
113
|
DATE_TIME = "Date/Time",
|
|
114
|
-
Boolean = "Boolean",// eslint-disable-line id-
|
|
114
|
+
Boolean = "Boolean",// eslint-disable-line id-denylist
|
|
115
115
|
NULL = "NULL"
|
|
116
116
|
}
|
|
117
117
|
export interface DatastorePropertyStats {
|
package/dist/query.util.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type Query } from '@google-cloud/datastore';
|
|
2
2
|
import type { DBQuery } from '@naturalcycles/db-lib';
|
|
3
3
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
4
|
-
export declare function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query
|
|
4
|
+
export declare function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query): Query;
|
package/dist/query.util.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { PropertyFilter } from '@google-cloud/datastore';
|
|
1
2
|
const FNAME_MAP = {
|
|
2
3
|
id: '__key__',
|
|
3
4
|
};
|
|
@@ -6,7 +7,7 @@ const OP_MAP = {
|
|
|
6
7
|
in: 'IN',
|
|
7
8
|
'not-in': 'NOT_IN',
|
|
8
9
|
};
|
|
9
|
-
export function dbQueryToDatastoreQuery(dbQuery, emptyQuery
|
|
10
|
+
export function dbQueryToDatastoreQuery(dbQuery, emptyQuery) {
|
|
10
11
|
let q = emptyQuery;
|
|
11
12
|
// filter
|
|
12
13
|
for (const f of dbQuery._filters) {
|
|
@@ -20,7 +21,7 @@ export function dbQueryToDatastoreQuery(dbQuery, emptyQuery, propertyFilterClass
|
|
|
20
21
|
let { op, val } = f;
|
|
21
22
|
if (val === undefined)
|
|
22
23
|
val = null;
|
|
23
|
-
q = q.filter(new
|
|
24
|
+
q = q.filter(new PropertyFilter(f.name, OP_MAP[op] || op, val));
|
|
24
25
|
}
|
|
25
26
|
// limit
|
|
26
27
|
q = q.limit(dbQuery._limitValue || 0);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/datastore-lib",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "4.
|
|
4
|
+
"version": "4.17.0",
|
|
5
5
|
"description": "Opinionated library to work with Google Datastore, implements CommonDB",
|
|
6
6
|
"dependencies": {
|
|
7
7
|
"@google-cloud/datastore": "^10",
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
},
|
|
12
12
|
"devDependencies": {
|
|
13
13
|
"@types/node": "^24",
|
|
14
|
-
"@naturalcycles/dev-lib": "
|
|
14
|
+
"@naturalcycles/dev-lib": "20.6.0"
|
|
15
15
|
},
|
|
16
16
|
"exports": {
|
|
17
17
|
".": "./dist/index.js"
|
package/src/datastore.db.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Key, Query, Transaction } from '@google-cloud/datastore'
|
|
2
|
+
import { Datastore, PropertyFilter } from '@google-cloud/datastore'
|
|
2
3
|
import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
|
|
3
4
|
import type {
|
|
4
5
|
CommonDB,
|
|
@@ -16,6 +17,7 @@ import type {
|
|
|
16
17
|
import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib'
|
|
17
18
|
import { _round } from '@naturalcycles/js-lib'
|
|
18
19
|
import { _chunk } from '@naturalcycles/js-lib/array/array.util.js'
|
|
20
|
+
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
|
|
19
21
|
import { _assert } from '@naturalcycles/js-lib/error/assert.js'
|
|
20
22
|
import { _errorDataAppend, TimeoutError } from '@naturalcycles/js-lib/error/error.util.js'
|
|
21
23
|
import type {
|
|
@@ -112,14 +114,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
112
114
|
protected KEY!: symbol
|
|
113
115
|
|
|
114
116
|
// @memo() // not used to be able to connect to many DBs in the same server instance
|
|
115
|
-
|
|
117
|
+
ds(): Datastore {
|
|
116
118
|
if (!this.cachedDatastore) {
|
|
117
119
|
_assert(
|
|
118
120
|
process.env['APP_ENV'] !== 'test',
|
|
119
121
|
'DatastoreDB cannot be used in Test env, please use InMemoryDB',
|
|
120
122
|
)
|
|
121
123
|
|
|
122
|
-
const DS = (await this.getDatastoreLib()).Datastore as typeof Datastore
|
|
123
124
|
this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT']
|
|
124
125
|
|
|
125
126
|
if (this.cfg.projectId) {
|
|
@@ -132,23 +133,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
132
133
|
this.cfg.logger.log('!!! DatastoreDB using custom grpc !!!')
|
|
133
134
|
}
|
|
134
135
|
|
|
135
|
-
this.cachedDatastore = new
|
|
136
|
+
this.cachedDatastore = new Datastore(this.cfg)
|
|
136
137
|
this.KEY = this.cachedDatastore.KEY
|
|
137
138
|
}
|
|
138
139
|
|
|
139
140
|
return this.cachedDatastore
|
|
140
141
|
}
|
|
141
142
|
|
|
142
|
-
private async getPropertyFilter(): Promise<typeof PropertyFilter> {
|
|
143
|
-
return (await this.getDatastoreLib()).PropertyFilter
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
private async getDatastoreLib(): Promise<any> {
|
|
147
|
-
// Lazy-loading
|
|
148
|
-
const lib = await import('@google-cloud/datastore')
|
|
149
|
-
return lib
|
|
150
|
-
}
|
|
151
|
-
|
|
152
143
|
override async ping(): Promise<void> {
|
|
153
144
|
await this.getAllStats()
|
|
154
145
|
}
|
|
@@ -159,7 +150,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
159
150
|
opt: DatastoreDBReadOptions = {},
|
|
160
151
|
): Promise<ROW[]> {
|
|
161
152
|
if (!ids.length) return []
|
|
162
|
-
let ds =
|
|
153
|
+
let ds = this.ds()
|
|
163
154
|
const keys = ids.map(id => this.key(ds, table, id))
|
|
164
155
|
let rows: any[]
|
|
165
156
|
|
|
@@ -182,12 +173,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
182
173
|
throw err
|
|
183
174
|
}
|
|
184
175
|
|
|
185
|
-
this.cfg.logger.log(
|
|
176
|
+
this.cfg.logger.log(
|
|
177
|
+
`datastore recreated on timeout (${_ms(this.cfg.timeout)}) while loading ${table}`,
|
|
178
|
+
)
|
|
186
179
|
|
|
187
180
|
// This is to debug "GCP Datastore Timeout issue"
|
|
188
|
-
|
|
189
|
-
const DS = datastoreLib.Datastore as typeof Datastore
|
|
190
|
-
ds = this.cachedDatastore = new DS(this.cfg)
|
|
181
|
+
ds = this.cachedDatastore = new Datastore(this.cfg)
|
|
191
182
|
|
|
192
183
|
// Second try (will throw)
|
|
193
184
|
try {
|
|
@@ -232,7 +223,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
232
223
|
opt: DatastoreDBReadOptions = {},
|
|
233
224
|
): Promise<StringMap<ROW[]>> {
|
|
234
225
|
const result: StringMap<ROW[]> = {}
|
|
235
|
-
const ds =
|
|
226
|
+
const ds = this.ds()
|
|
236
227
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
237
228
|
const keys: Key[] = []
|
|
238
229
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
@@ -275,12 +266,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
275
266
|
}
|
|
276
267
|
}
|
|
277
268
|
|
|
278
|
-
const ds =
|
|
279
|
-
const q = dbQueryToDatastoreQuery(
|
|
280
|
-
dbQuery,
|
|
281
|
-
ds.createQuery(dbQuery.table),
|
|
282
|
-
await this.getPropertyFilter(),
|
|
283
|
-
)
|
|
269
|
+
const ds = this.ds()
|
|
270
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
284
271
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
285
272
|
const qr = await this.runDatastoreQuery<ROW>(q, dsOpt)
|
|
286
273
|
|
|
@@ -296,12 +283,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
296
283
|
dbQuery: DBQuery<ROW>,
|
|
297
284
|
opt: DatastoreDBReadOptions = {},
|
|
298
285
|
): Promise<number> {
|
|
299
|
-
const ds =
|
|
300
|
-
const q = dbQueryToDatastoreQuery(
|
|
301
|
-
dbQuery,
|
|
302
|
-
ds.createQuery(dbQuery.table),
|
|
303
|
-
await this.getPropertyFilter(),
|
|
304
|
-
)
|
|
286
|
+
const ds = this.ds()
|
|
287
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
305
288
|
const aq = ds.createAggregationQuery(q).count('count')
|
|
306
289
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
307
290
|
const [entities] = await ds.runAggregationQuery(aq, dsOpt)
|
|
@@ -312,7 +295,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
312
295
|
q: Query,
|
|
313
296
|
dsOpt: RunQueryOptions,
|
|
314
297
|
): Promise<RunQueryResult<ROW>> {
|
|
315
|
-
const ds =
|
|
298
|
+
const ds = this.ds()
|
|
316
299
|
const [entities, queryResult] = await ds.runQuery(q, dsOpt)
|
|
317
300
|
|
|
318
301
|
const rows = entities.map(e => this.mapId<ROW>(e))
|
|
@@ -328,13 +311,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
328
311
|
_opt?: DatastoreDBStreamOptions,
|
|
329
312
|
): Pipeline<ROW> {
|
|
330
313
|
return Pipeline.fromAsyncReadable<ROW>(async () => {
|
|
331
|
-
const ds =
|
|
314
|
+
const ds = this.ds()
|
|
332
315
|
|
|
333
|
-
const q = dbQueryToDatastoreQuery(
|
|
334
|
-
dbQuery,
|
|
335
|
-
ds.createQuery(dbQuery.table),
|
|
336
|
-
await this.getPropertyFilter(),
|
|
337
|
-
)
|
|
316
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
338
317
|
|
|
339
318
|
const opt = {
|
|
340
319
|
logger: this.cfg.logger,
|
|
@@ -358,7 +337,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
358
337
|
rows: ROW[],
|
|
359
338
|
opt: DatastoreDBSaveOptions<ROW> = {},
|
|
360
339
|
): Promise<void> {
|
|
361
|
-
const ds =
|
|
340
|
+
const ds = this.ds()
|
|
362
341
|
const entities = rows.map(obj =>
|
|
363
342
|
this.toDatastoreEntity(ds, table, obj, opt.excludeFromIndexes as string[]),
|
|
364
343
|
)
|
|
@@ -411,12 +390,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
411
390
|
return await this.deleteByIds(q.table, ids, opt)
|
|
412
391
|
}
|
|
413
392
|
|
|
414
|
-
const ds =
|
|
415
|
-
const datastoreQuery = dbQueryToDatastoreQuery(
|
|
416
|
-
q.select([]),
|
|
417
|
-
ds.createQuery(q.table),
|
|
418
|
-
await this.getPropertyFilter(),
|
|
419
|
-
)
|
|
393
|
+
const ds = this.ds()
|
|
394
|
+
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table))
|
|
420
395
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
421
396
|
const { rows } = await this.runDatastoreQuery<ObjectWithId>(datastoreQuery, dsOpt)
|
|
422
397
|
return await this.deleteByIds(
|
|
@@ -435,7 +410,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
435
410
|
ids: string[],
|
|
436
411
|
opt: DatastoreDBOptions = {},
|
|
437
412
|
): Promise<number> {
|
|
438
|
-
const ds =
|
|
413
|
+
const ds = this.ds()
|
|
439
414
|
const keys = ids.map(id => this.key(ds, table, id))
|
|
440
415
|
|
|
441
416
|
const retryOptions = this.getPRetryOptions(`DatastoreLib.deleteByIds(${table})`)
|
|
@@ -459,7 +434,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
459
434
|
map: StringMap<string[]>,
|
|
460
435
|
opt: DatastoreDBOptions = {},
|
|
461
436
|
): Promise<number> {
|
|
462
|
-
const ds =
|
|
437
|
+
const ds = this.ds()
|
|
463
438
|
const keys: Key[] = []
|
|
464
439
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
465
440
|
keys.push(...ids.map(id => this.key(ds, table, id)))
|
|
@@ -486,7 +461,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
486
461
|
override async createTransaction(
|
|
487
462
|
opt: CommonDBTransactionOptions = {},
|
|
488
463
|
): Promise<DatastoreDBTransaction> {
|
|
489
|
-
const ds =
|
|
464
|
+
const ds = this.ds()
|
|
490
465
|
const { readOnly } = opt
|
|
491
466
|
const datastoreTx = ds.transaction({
|
|
492
467
|
readOnly,
|
|
@@ -499,7 +474,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
499
474
|
fn: DBTransactionFn,
|
|
500
475
|
opt: CommonDBTransactionOptions = {},
|
|
501
476
|
): Promise<void> {
|
|
502
|
-
const ds =
|
|
477
|
+
const ds = this.ds()
|
|
503
478
|
const { readOnly } = opt
|
|
504
479
|
const datastoreTx = ds.transaction({
|
|
505
480
|
readOnly,
|
|
@@ -517,7 +492,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
517
492
|
}
|
|
518
493
|
|
|
519
494
|
async getAllStats(): Promise<DatastoreStats[]> {
|
|
520
|
-
const ds =
|
|
495
|
+
const ds = this.ds()
|
|
521
496
|
const q = ds.createQuery('__Stat_Kind__')
|
|
522
497
|
const [statsArray] = await ds.runQuery(q)
|
|
523
498
|
return statsArray || []
|
|
@@ -527,13 +502,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
527
502
|
* Returns undefined e.g when Table is non-existing
|
|
528
503
|
*/
|
|
529
504
|
async getStats(table: string): Promise<DatastoreStats | undefined> {
|
|
530
|
-
const ds =
|
|
531
|
-
const propertyFilter = await this.getPropertyFilter()
|
|
505
|
+
const ds = this.ds()
|
|
532
506
|
|
|
533
507
|
const q = ds
|
|
534
508
|
.createQuery('__Stat_Kind__')
|
|
535
509
|
// .filter('kind_name', table)
|
|
536
|
-
.filter(new
|
|
510
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
537
511
|
.limit(1)
|
|
538
512
|
const [statsArray] = await ds.runQuery(q)
|
|
539
513
|
const [stats] = statsArray
|
|
@@ -546,11 +520,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
546
520
|
}
|
|
547
521
|
|
|
548
522
|
async getTableProperties(table: string): Promise<DatastorePropertyStats[]> {
|
|
549
|
-
const ds =
|
|
523
|
+
const ds = this.ds()
|
|
550
524
|
const q = ds
|
|
551
525
|
.createQuery('__Stat_PropertyType_PropertyName_Kind__')
|
|
552
526
|
// .filter('kind_name', table)
|
|
553
|
-
.filter(new (
|
|
527
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
554
528
|
const [stats] = await ds.runQuery(q)
|
|
555
529
|
return stats
|
|
556
530
|
}
|
package/src/datastore.model.ts
CHANGED
|
@@ -128,12 +128,12 @@ export interface DatastoreStats {
|
|
|
128
128
|
export enum DatastoreType {
|
|
129
129
|
Blob = 'Blob',
|
|
130
130
|
Text = 'Text',
|
|
131
|
-
String = 'String', // eslint-disable-line id-
|
|
131
|
+
String = 'String', // eslint-disable-line id-denylist
|
|
132
132
|
EmbeddedEntity = 'EmbeddedEntity',
|
|
133
133
|
Float = 'Float',
|
|
134
134
|
Integer = 'Integer',
|
|
135
135
|
DATE_TIME = 'Date/Time',
|
|
136
|
-
Boolean = 'Boolean', // eslint-disable-line id-
|
|
136
|
+
Boolean = 'Boolean', // eslint-disable-line id-denylist
|
|
137
137
|
NULL = 'NULL',
|
|
138
138
|
}
|
|
139
139
|
|
package/src/query.util.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { PropertyFilter, type Query } from '@google-cloud/datastore'
|
|
2
2
|
import type { DBQuery, DBQueryFilterOperator } from '@naturalcycles/db-lib'
|
|
3
3
|
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
|
|
4
4
|
|
|
@@ -15,7 +15,6 @@ const OP_MAP: Partial<Record<DBQueryFilterOperator, string>> = {
|
|
|
15
15
|
export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
|
|
16
16
|
dbQuery: Readonly<DBQuery<ROW>>,
|
|
17
17
|
emptyQuery: Query,
|
|
18
|
-
propertyFilterClass: typeof PropertyFilter,
|
|
19
18
|
): Query {
|
|
20
19
|
let q = emptyQuery
|
|
21
20
|
|
|
@@ -31,7 +30,7 @@ export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
|
|
|
31
30
|
// `a == null` will return just that - rows with null values
|
|
32
31
|
let { op, val } = f
|
|
33
32
|
if (val === undefined) val = null
|
|
34
|
-
q = q.filter(new
|
|
33
|
+
q = q.filter(new PropertyFilter(f.name as string, OP_MAP[op] || (op as any), val))
|
|
35
34
|
}
|
|
36
35
|
|
|
37
36
|
// limit
|