@naturalcycles/datastore-lib 4.16.1 → 4.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/datastore.db.d.ts +6 -7
- package/dist/datastore.db.js +25 -36
- package/dist/datastore.model.d.ts +2 -2
- package/dist/query.util.d.ts +2 -2
- package/dist/query.util.js +3 -2
- package/package.json +1 -1
- package/src/datastore.db.ts +46 -85
- package/src/datastore.model.ts +2 -2
- package/src/query.util.ts +2 -3
package/dist/datastore.db.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Key, Transaction } from '@google-cloud/datastore';
|
|
2
|
+
import { Datastore } from '@google-cloud/datastore';
|
|
2
3
|
import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBSupport, CommonDBTransactionOptions, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
|
|
3
4
|
import { BaseCommonDB } from '@naturalcycles/db-lib';
|
|
4
|
-
import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
|
|
5
5
|
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
6
6
|
import { type ObjectWithId, type StringMap } from '@naturalcycles/js-lib/types';
|
|
7
|
+
import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv';
|
|
7
8
|
import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
|
|
8
9
|
import type { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBReadOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePropertyStats, DatastoreStats } from './datastore.model.js';
|
|
9
10
|
/**
|
|
@@ -22,9 +23,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
22
23
|
* Datastore.KEY
|
|
23
24
|
*/
|
|
24
25
|
protected KEY: symbol;
|
|
25
|
-
ds():
|
|
26
|
-
private getPropertyFilter;
|
|
27
|
-
private getDatastoreLib;
|
|
26
|
+
ds(): Datastore;
|
|
28
27
|
ping(): Promise<void>;
|
|
29
28
|
getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: DatastoreDBReadOptions): Promise<ROW[]>;
|
|
30
29
|
multiGet<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: DatastoreDBReadOptions): Promise<StringMap<ROW[]>>;
|
|
@@ -58,9 +57,9 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
58
57
|
key(ds: Datastore, kind: string, id: string): Key;
|
|
59
58
|
private getDsKey;
|
|
60
59
|
private getIdFromKey;
|
|
61
|
-
createTable<ROW extends ObjectWithId>(_table: string, _schema:
|
|
60
|
+
createTable<ROW extends ObjectWithId>(_table: string, _schema: JsonSchema<ROW>): Promise<void>;
|
|
62
61
|
getTables(): Promise<string[]>;
|
|
63
|
-
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<
|
|
62
|
+
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchema<ROW>>;
|
|
64
63
|
private getPRetryOptions;
|
|
65
64
|
/**
|
|
66
65
|
* Silently rollback the transaction.
|
package/dist/datastore.db.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Datastore, PropertyFilter } from '@google-cloud/datastore';
|
|
1
2
|
import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
|
|
2
3
|
import { _round } from '@naturalcycles/js-lib';
|
|
3
4
|
import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
|
|
@@ -62,10 +63,9 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
62
63
|
*/
|
|
63
64
|
KEY;
|
|
64
65
|
// @memo() // not used to be able to connect to many DBs in the same server instance
|
|
65
|
-
|
|
66
|
+
ds() {
|
|
66
67
|
if (!this.cachedDatastore) {
|
|
67
68
|
_assert(process.env['APP_ENV'] !== 'test', 'DatastoreDB cannot be used in Test env, please use InMemoryDB');
|
|
68
|
-
const DS = (await this.getDatastoreLib()).Datastore;
|
|
69
69
|
this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT'];
|
|
70
70
|
if (this.cfg.projectId) {
|
|
71
71
|
this.cfg.logger.log(`DatastoreDB connected to ${boldWhite(this.cfg.projectId)}`);
|
|
@@ -76,26 +76,18 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
76
76
|
if (this.cfg.grpc) {
|
|
77
77
|
this.cfg.logger.log('!!! DatastoreDB using custom grpc !!!');
|
|
78
78
|
}
|
|
79
|
-
this.cachedDatastore = new
|
|
79
|
+
this.cachedDatastore = new Datastore(this.cfg);
|
|
80
80
|
this.KEY = this.cachedDatastore.KEY;
|
|
81
81
|
}
|
|
82
82
|
return this.cachedDatastore;
|
|
83
83
|
}
|
|
84
|
-
async getPropertyFilter() {
|
|
85
|
-
return (await this.getDatastoreLib()).PropertyFilter;
|
|
86
|
-
}
|
|
87
|
-
async getDatastoreLib() {
|
|
88
|
-
// Lazy-loading
|
|
89
|
-
const lib = await import('@google-cloud/datastore');
|
|
90
|
-
return lib;
|
|
91
|
-
}
|
|
92
84
|
async ping() {
|
|
93
85
|
await this.getAllStats();
|
|
94
86
|
}
|
|
95
87
|
async getByIds(table, ids, opt = {}) {
|
|
96
88
|
if (!ids.length)
|
|
97
89
|
return [];
|
|
98
|
-
let ds =
|
|
90
|
+
let ds = this.ds();
|
|
99
91
|
const keys = ids.map(id => this.key(ds, table, id));
|
|
100
92
|
let rows;
|
|
101
93
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
@@ -115,9 +107,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
115
107
|
}
|
|
116
108
|
this.cfg.logger.log(`datastore recreated on timeout (${_ms(this.cfg.timeout)}) while loading ${table}`);
|
|
117
109
|
// This is to debug "GCP Datastore Timeout issue"
|
|
118
|
-
|
|
119
|
-
const DS = datastoreLib.Datastore;
|
|
120
|
-
ds = this.cachedDatastore = new DS(this.cfg);
|
|
110
|
+
ds = this.cachedDatastore = new Datastore(this.cfg);
|
|
121
111
|
// Second try (will throw)
|
|
122
112
|
try {
|
|
123
113
|
const r = await pRetry(() => (opt.tx?.tx || ds).get(keys, dsOpt), {
|
|
@@ -150,7 +140,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
150
140
|
}
|
|
151
141
|
async multiGet(map, opt = {}) {
|
|
152
142
|
const result = {};
|
|
153
|
-
const ds =
|
|
143
|
+
const ds = this.ds();
|
|
154
144
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
155
145
|
const keys = [];
|
|
156
146
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
@@ -182,8 +172,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
182
172
|
rows: await this.getByIds(dbQuery.table, ids, opt),
|
|
183
173
|
};
|
|
184
174
|
}
|
|
185
|
-
const ds =
|
|
186
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
175
|
+
const ds = this.ds();
|
|
176
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
187
177
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
188
178
|
const qr = await this.runDatastoreQuery(q, dsOpt);
|
|
189
179
|
// Special case when projection query didn't specify 'id'
|
|
@@ -193,15 +183,15 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
193
183
|
return qr;
|
|
194
184
|
}
|
|
195
185
|
async runQueryCount(dbQuery, opt = {}) {
|
|
196
|
-
const ds =
|
|
197
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
186
|
+
const ds = this.ds();
|
|
187
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
198
188
|
const aq = ds.createAggregationQuery(q).count('count');
|
|
199
189
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
200
190
|
const [entities] = await ds.runAggregationQuery(aq, dsOpt);
|
|
201
191
|
return entities[0]?.count;
|
|
202
192
|
}
|
|
203
193
|
async runDatastoreQuery(q, dsOpt) {
|
|
204
|
-
const ds =
|
|
194
|
+
const ds = this.ds();
|
|
205
195
|
const [entities, queryResult] = await ds.runQuery(q, dsOpt);
|
|
206
196
|
const rows = entities.map(e => this.mapId(e));
|
|
207
197
|
return {
|
|
@@ -211,8 +201,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
211
201
|
}
|
|
212
202
|
streamQuery(dbQuery, _opt) {
|
|
213
203
|
return Pipeline.fromAsyncReadable(async () => {
|
|
214
|
-
const ds =
|
|
215
|
-
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table)
|
|
204
|
+
const ds = this.ds();
|
|
205
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
|
|
216
206
|
const opt = {
|
|
217
207
|
logger: this.cfg.logger,
|
|
218
208
|
...this.cfg.streamOptions,
|
|
@@ -228,7 +218,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
228
218
|
* Returns saved entities with generated id/updated/created (non-mutating!)
|
|
229
219
|
*/
|
|
230
220
|
async saveBatch(table, rows, opt = {}) {
|
|
231
|
-
const ds =
|
|
221
|
+
const ds = this.ds();
|
|
232
222
|
const entities = rows.map(obj => this.toDatastoreEntity(ds, table, obj, opt.excludeFromIndexes));
|
|
233
223
|
const method = methodMap[opt.saveMethod || 'upsert'] || 'save';
|
|
234
224
|
const save = pRetryFn(async (batch) => {
|
|
@@ -264,8 +254,8 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
264
254
|
const ids = idFilter.op === '==' ? [idFilter.val] : idFilter.val;
|
|
265
255
|
return await this.deleteByIds(q.table, ids, opt);
|
|
266
256
|
}
|
|
267
|
-
const ds =
|
|
268
|
-
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table)
|
|
257
|
+
const ds = this.ds();
|
|
258
|
+
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table));
|
|
269
259
|
const dsOpt = this.getRunQueryOptions(opt);
|
|
270
260
|
const { rows } = await this.runDatastoreQuery(datastoreQuery, dsOpt);
|
|
271
261
|
return await this.deleteByIds(q.table, rows.map(obj => obj.id), opt);
|
|
@@ -275,7 +265,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
275
265
|
* regardless if they were actually deleted or not.
|
|
276
266
|
*/
|
|
277
267
|
async deleteByIds(table, ids, opt = {}) {
|
|
278
|
-
const ds =
|
|
268
|
+
const ds = this.ds();
|
|
279
269
|
const keys = ids.map(id => this.key(ds, table, id));
|
|
280
270
|
const retryOptions = this.getPRetryOptions(`DatastoreLib.deleteByIds(${table})`);
|
|
281
271
|
await pMap(_chunk(keys, MAX_ITEMS),
|
|
@@ -290,7 +280,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
290
280
|
return ids.length;
|
|
291
281
|
}
|
|
292
282
|
async multiDelete(map, opt = {}) {
|
|
293
|
-
const ds =
|
|
283
|
+
const ds = this.ds();
|
|
294
284
|
const keys = [];
|
|
295
285
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
296
286
|
keys.push(...ids.map(id => this.key(ds, table, id)));
|
|
@@ -308,7 +298,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
308
298
|
return keys.length;
|
|
309
299
|
}
|
|
310
300
|
async createTransaction(opt = {}) {
|
|
311
|
-
const ds =
|
|
301
|
+
const ds = this.ds();
|
|
312
302
|
const { readOnly } = opt;
|
|
313
303
|
const datastoreTx = ds.transaction({
|
|
314
304
|
readOnly,
|
|
@@ -317,7 +307,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
317
307
|
return new DatastoreDBTransaction(this, datastoreTx);
|
|
318
308
|
}
|
|
319
309
|
async runInTransaction(fn, opt = {}) {
|
|
320
|
-
const ds =
|
|
310
|
+
const ds = this.ds();
|
|
321
311
|
const { readOnly } = opt;
|
|
322
312
|
const datastoreTx = ds.transaction({
|
|
323
313
|
readOnly,
|
|
@@ -334,7 +324,7 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
334
324
|
}
|
|
335
325
|
}
|
|
336
326
|
async getAllStats() {
|
|
337
|
-
const ds =
|
|
327
|
+
const ds = this.ds();
|
|
338
328
|
const q = ds.createQuery('__Stat_Kind__');
|
|
339
329
|
const [statsArray] = await ds.runQuery(q);
|
|
340
330
|
return statsArray || [];
|
|
@@ -343,12 +333,11 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
343
333
|
* Returns undefined e.g when Table is non-existing
|
|
344
334
|
*/
|
|
345
335
|
async getStats(table) {
|
|
346
|
-
const ds =
|
|
347
|
-
const propertyFilter = await this.getPropertyFilter();
|
|
336
|
+
const ds = this.ds();
|
|
348
337
|
const q = ds
|
|
349
338
|
.createQuery('__Stat_Kind__')
|
|
350
339
|
// .filter('kind_name', table)
|
|
351
|
-
.filter(new
|
|
340
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
352
341
|
.limit(1);
|
|
353
342
|
const [statsArray] = await ds.runQuery(q);
|
|
354
343
|
const [stats] = statsArray;
|
|
@@ -359,11 +348,11 @@ export class DatastoreDB extends BaseCommonDB {
|
|
|
359
348
|
return stats?.count;
|
|
360
349
|
}
|
|
361
350
|
async getTableProperties(table) {
|
|
362
|
-
const ds =
|
|
351
|
+
const ds = this.ds();
|
|
363
352
|
const q = ds
|
|
364
353
|
.createQuery('__Stat_PropertyType_PropertyName_Kind__')
|
|
365
354
|
// .filter('kind_name', table)
|
|
366
|
-
.filter(new (
|
|
355
|
+
.filter(new PropertyFilter('kind_name', '=', table));
|
|
367
356
|
const [stats] = await ds.runQuery(q);
|
|
368
357
|
return stats;
|
|
369
358
|
}
|
|
@@ -106,12 +106,12 @@ export interface DatastoreStats {
|
|
|
106
106
|
export declare enum DatastoreType {
|
|
107
107
|
Blob = "Blob",
|
|
108
108
|
Text = "Text",
|
|
109
|
-
String = "String",// eslint-disable-line id-
|
|
109
|
+
String = "String",// eslint-disable-line id-denylist
|
|
110
110
|
EmbeddedEntity = "EmbeddedEntity",
|
|
111
111
|
Float = "Float",
|
|
112
112
|
Integer = "Integer",
|
|
113
113
|
DATE_TIME = "Date/Time",
|
|
114
|
-
Boolean = "Boolean",// eslint-disable-line id-
|
|
114
|
+
Boolean = "Boolean",// eslint-disable-line id-denylist
|
|
115
115
|
NULL = "NULL"
|
|
116
116
|
}
|
|
117
117
|
export interface DatastorePropertyStats {
|
package/dist/query.util.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { type Query } from '@google-cloud/datastore';
|
|
2
2
|
import type { DBQuery } from '@naturalcycles/db-lib';
|
|
3
3
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
4
|
-
export declare function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query
|
|
4
|
+
export declare function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query): Query;
|
package/dist/query.util.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { PropertyFilter } from '@google-cloud/datastore';
|
|
1
2
|
const FNAME_MAP = {
|
|
2
3
|
id: '__key__',
|
|
3
4
|
};
|
|
@@ -6,7 +7,7 @@ const OP_MAP = {
|
|
|
6
7
|
in: 'IN',
|
|
7
8
|
'not-in': 'NOT_IN',
|
|
8
9
|
};
|
|
9
|
-
export function dbQueryToDatastoreQuery(dbQuery, emptyQuery
|
|
10
|
+
export function dbQueryToDatastoreQuery(dbQuery, emptyQuery) {
|
|
10
11
|
let q = emptyQuery;
|
|
11
12
|
// filter
|
|
12
13
|
for (const f of dbQuery._filters) {
|
|
@@ -20,7 +21,7 @@ export function dbQueryToDatastoreQuery(dbQuery, emptyQuery, propertyFilterClass
|
|
|
20
21
|
let { op, val } = f;
|
|
21
22
|
if (val === undefined)
|
|
22
23
|
val = null;
|
|
23
|
-
q = q.filter(new
|
|
24
|
+
q = q.filter(new PropertyFilter(f.name, OP_MAP[op] || op, val));
|
|
24
25
|
}
|
|
25
26
|
// limit
|
|
26
27
|
q = q.limit(dbQuery._limitValue || 0);
|
package/package.json
CHANGED
package/src/datastore.db.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Key, Query, Transaction } from '@google-cloud/datastore'
|
|
2
|
+
import { Datastore, PropertyFilter } from '@google-cloud/datastore'
|
|
2
3
|
import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
|
|
3
4
|
import type {
|
|
4
5
|
CommonDB,
|
|
@@ -19,15 +20,6 @@ import { _chunk } from '@naturalcycles/js-lib/array/array.util.js'
|
|
|
19
20
|
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
|
|
20
21
|
import { _assert } from '@naturalcycles/js-lib/error/assert.js'
|
|
21
22
|
import { _errorDataAppend, TimeoutError } from '@naturalcycles/js-lib/error/error.util.js'
|
|
22
|
-
import type {
|
|
23
|
-
JsonSchemaAny,
|
|
24
|
-
JsonSchemaBoolean,
|
|
25
|
-
JsonSchemaNull,
|
|
26
|
-
JsonSchemaNumber,
|
|
27
|
-
JsonSchemaObject,
|
|
28
|
-
JsonSchemaRootObject,
|
|
29
|
-
JsonSchemaString,
|
|
30
|
-
} from '@naturalcycles/js-lib/json-schema'
|
|
31
23
|
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
32
24
|
import { _omit } from '@naturalcycles/js-lib/object/object.util.js'
|
|
33
25
|
import type { PRetryOptions } from '@naturalcycles/js-lib/promise'
|
|
@@ -40,6 +32,7 @@ import {
|
|
|
40
32
|
type ObjectWithId,
|
|
41
33
|
type StringMap,
|
|
42
34
|
} from '@naturalcycles/js-lib/types'
|
|
35
|
+
import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv'
|
|
43
36
|
import { boldWhite } from '@naturalcycles/nodejs-lib/colors'
|
|
44
37
|
import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
|
|
45
38
|
import type {
|
|
@@ -113,14 +106,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
113
106
|
protected KEY!: symbol
|
|
114
107
|
|
|
115
108
|
// @memo() // not used to be able to connect to many DBs in the same server instance
|
|
116
|
-
|
|
109
|
+
ds(): Datastore {
|
|
117
110
|
if (!this.cachedDatastore) {
|
|
118
111
|
_assert(
|
|
119
112
|
process.env['APP_ENV'] !== 'test',
|
|
120
113
|
'DatastoreDB cannot be used in Test env, please use InMemoryDB',
|
|
121
114
|
)
|
|
122
115
|
|
|
123
|
-
const DS = (await this.getDatastoreLib()).Datastore as typeof Datastore
|
|
124
116
|
this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT']
|
|
125
117
|
|
|
126
118
|
if (this.cfg.projectId) {
|
|
@@ -133,23 +125,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
133
125
|
this.cfg.logger.log('!!! DatastoreDB using custom grpc !!!')
|
|
134
126
|
}
|
|
135
127
|
|
|
136
|
-
this.cachedDatastore = new
|
|
128
|
+
this.cachedDatastore = new Datastore(this.cfg)
|
|
137
129
|
this.KEY = this.cachedDatastore.KEY
|
|
138
130
|
}
|
|
139
131
|
|
|
140
132
|
return this.cachedDatastore
|
|
141
133
|
}
|
|
142
134
|
|
|
143
|
-
private async getPropertyFilter(): Promise<typeof PropertyFilter> {
|
|
144
|
-
return (await this.getDatastoreLib()).PropertyFilter
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
private async getDatastoreLib(): Promise<any> {
|
|
148
|
-
// Lazy-loading
|
|
149
|
-
const lib = await import('@google-cloud/datastore')
|
|
150
|
-
return lib
|
|
151
|
-
}
|
|
152
|
-
|
|
153
135
|
override async ping(): Promise<void> {
|
|
154
136
|
await this.getAllStats()
|
|
155
137
|
}
|
|
@@ -160,7 +142,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
160
142
|
opt: DatastoreDBReadOptions = {},
|
|
161
143
|
): Promise<ROW[]> {
|
|
162
144
|
if (!ids.length) return []
|
|
163
|
-
let ds =
|
|
145
|
+
let ds = this.ds()
|
|
164
146
|
const keys = ids.map(id => this.key(ds, table, id))
|
|
165
147
|
let rows: any[]
|
|
166
148
|
|
|
@@ -188,9 +170,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
188
170
|
)
|
|
189
171
|
|
|
190
172
|
// This is to debug "GCP Datastore Timeout issue"
|
|
191
|
-
|
|
192
|
-
const DS = datastoreLib.Datastore as typeof Datastore
|
|
193
|
-
ds = this.cachedDatastore = new DS(this.cfg)
|
|
173
|
+
ds = this.cachedDatastore = new Datastore(this.cfg)
|
|
194
174
|
|
|
195
175
|
// Second try (will throw)
|
|
196
176
|
try {
|
|
@@ -235,7 +215,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
235
215
|
opt: DatastoreDBReadOptions = {},
|
|
236
216
|
): Promise<StringMap<ROW[]>> {
|
|
237
217
|
const result: StringMap<ROW[]> = {}
|
|
238
|
-
const ds =
|
|
218
|
+
const ds = this.ds()
|
|
239
219
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
240
220
|
const keys: Key[] = []
|
|
241
221
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
@@ -278,12 +258,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
278
258
|
}
|
|
279
259
|
}
|
|
280
260
|
|
|
281
|
-
const ds =
|
|
282
|
-
const q = dbQueryToDatastoreQuery(
|
|
283
|
-
dbQuery,
|
|
284
|
-
ds.createQuery(dbQuery.table),
|
|
285
|
-
await this.getPropertyFilter(),
|
|
286
|
-
)
|
|
261
|
+
const ds = this.ds()
|
|
262
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
287
263
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
288
264
|
const qr = await this.runDatastoreQuery<ROW>(q, dsOpt)
|
|
289
265
|
|
|
@@ -299,12 +275,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
299
275
|
dbQuery: DBQuery<ROW>,
|
|
300
276
|
opt: DatastoreDBReadOptions = {},
|
|
301
277
|
): Promise<number> {
|
|
302
|
-
const ds =
|
|
303
|
-
const q = dbQueryToDatastoreQuery(
|
|
304
|
-
dbQuery,
|
|
305
|
-
ds.createQuery(dbQuery.table),
|
|
306
|
-
await this.getPropertyFilter(),
|
|
307
|
-
)
|
|
278
|
+
const ds = this.ds()
|
|
279
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
308
280
|
const aq = ds.createAggregationQuery(q).count('count')
|
|
309
281
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
310
282
|
const [entities] = await ds.runAggregationQuery(aq, dsOpt)
|
|
@@ -315,7 +287,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
315
287
|
q: Query,
|
|
316
288
|
dsOpt: RunQueryOptions,
|
|
317
289
|
): Promise<RunQueryResult<ROW>> {
|
|
318
|
-
const ds =
|
|
290
|
+
const ds = this.ds()
|
|
319
291
|
const [entities, queryResult] = await ds.runQuery(q, dsOpt)
|
|
320
292
|
|
|
321
293
|
const rows = entities.map(e => this.mapId<ROW>(e))
|
|
@@ -331,13 +303,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
331
303
|
_opt?: DatastoreDBStreamOptions,
|
|
332
304
|
): Pipeline<ROW> {
|
|
333
305
|
return Pipeline.fromAsyncReadable<ROW>(async () => {
|
|
334
|
-
const ds =
|
|
306
|
+
const ds = this.ds()
|
|
335
307
|
|
|
336
|
-
const q = dbQueryToDatastoreQuery(
|
|
337
|
-
dbQuery,
|
|
338
|
-
ds.createQuery(dbQuery.table),
|
|
339
|
-
await this.getPropertyFilter(),
|
|
340
|
-
)
|
|
308
|
+
const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
|
|
341
309
|
|
|
342
310
|
const opt = {
|
|
343
311
|
logger: this.cfg.logger,
|
|
@@ -361,7 +329,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
361
329
|
rows: ROW[],
|
|
362
330
|
opt: DatastoreDBSaveOptions<ROW> = {},
|
|
363
331
|
): Promise<void> {
|
|
364
|
-
const ds =
|
|
332
|
+
const ds = this.ds()
|
|
365
333
|
const entities = rows.map(obj =>
|
|
366
334
|
this.toDatastoreEntity(ds, table, obj, opt.excludeFromIndexes as string[]),
|
|
367
335
|
)
|
|
@@ -414,12 +382,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
414
382
|
return await this.deleteByIds(q.table, ids, opt)
|
|
415
383
|
}
|
|
416
384
|
|
|
417
|
-
const ds =
|
|
418
|
-
const datastoreQuery = dbQueryToDatastoreQuery(
|
|
419
|
-
q.select([]),
|
|
420
|
-
ds.createQuery(q.table),
|
|
421
|
-
await this.getPropertyFilter(),
|
|
422
|
-
)
|
|
385
|
+
const ds = this.ds()
|
|
386
|
+
const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table))
|
|
423
387
|
const dsOpt = this.getRunQueryOptions(opt)
|
|
424
388
|
const { rows } = await this.runDatastoreQuery<ObjectWithId>(datastoreQuery, dsOpt)
|
|
425
389
|
return await this.deleteByIds(
|
|
@@ -438,7 +402,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
438
402
|
ids: string[],
|
|
439
403
|
opt: DatastoreDBOptions = {},
|
|
440
404
|
): Promise<number> {
|
|
441
|
-
const ds =
|
|
405
|
+
const ds = this.ds()
|
|
442
406
|
const keys = ids.map(id => this.key(ds, table, id))
|
|
443
407
|
|
|
444
408
|
const retryOptions = this.getPRetryOptions(`DatastoreLib.deleteByIds(${table})`)
|
|
@@ -462,7 +426,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
462
426
|
map: StringMap<string[]>,
|
|
463
427
|
opt: DatastoreDBOptions = {},
|
|
464
428
|
): Promise<number> {
|
|
465
|
-
const ds =
|
|
429
|
+
const ds = this.ds()
|
|
466
430
|
const keys: Key[] = []
|
|
467
431
|
for (const [table, ids] of _stringMapEntries(map)) {
|
|
468
432
|
keys.push(...ids.map(id => this.key(ds, table, id)))
|
|
@@ -489,7 +453,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
489
453
|
override async createTransaction(
|
|
490
454
|
opt: CommonDBTransactionOptions = {},
|
|
491
455
|
): Promise<DatastoreDBTransaction> {
|
|
492
|
-
const ds =
|
|
456
|
+
const ds = this.ds()
|
|
493
457
|
const { readOnly } = opt
|
|
494
458
|
const datastoreTx = ds.transaction({
|
|
495
459
|
readOnly,
|
|
@@ -502,7 +466,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
502
466
|
fn: DBTransactionFn,
|
|
503
467
|
opt: CommonDBTransactionOptions = {},
|
|
504
468
|
): Promise<void> {
|
|
505
|
-
const ds =
|
|
469
|
+
const ds = this.ds()
|
|
506
470
|
const { readOnly } = opt
|
|
507
471
|
const datastoreTx = ds.transaction({
|
|
508
472
|
readOnly,
|
|
@@ -520,7 +484,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
520
484
|
}
|
|
521
485
|
|
|
522
486
|
async getAllStats(): Promise<DatastoreStats[]> {
|
|
523
|
-
const ds =
|
|
487
|
+
const ds = this.ds()
|
|
524
488
|
const q = ds.createQuery('__Stat_Kind__')
|
|
525
489
|
const [statsArray] = await ds.runQuery(q)
|
|
526
490
|
return statsArray || []
|
|
@@ -530,13 +494,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
530
494
|
* Returns undefined e.g when Table is non-existing
|
|
531
495
|
*/
|
|
532
496
|
async getStats(table: string): Promise<DatastoreStats | undefined> {
|
|
533
|
-
const ds =
|
|
534
|
-
const propertyFilter = await this.getPropertyFilter()
|
|
497
|
+
const ds = this.ds()
|
|
535
498
|
|
|
536
499
|
const q = ds
|
|
537
500
|
.createQuery('__Stat_Kind__')
|
|
538
501
|
// .filter('kind_name', table)
|
|
539
|
-
.filter(new
|
|
502
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
540
503
|
.limit(1)
|
|
541
504
|
const [statsArray] = await ds.runQuery(q)
|
|
542
505
|
const [stats] = statsArray
|
|
@@ -549,11 +512,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
549
512
|
}
|
|
550
513
|
|
|
551
514
|
async getTableProperties(table: string): Promise<DatastorePropertyStats[]> {
|
|
552
|
-
const ds =
|
|
515
|
+
const ds = this.ds()
|
|
553
516
|
const q = ds
|
|
554
517
|
.createQuery('__Stat_PropertyType_PropertyName_Kind__')
|
|
555
518
|
// .filter('kind_name', table)
|
|
556
|
-
.filter(new (
|
|
519
|
+
.filter(new PropertyFilter('kind_name', '=', table))
|
|
557
520
|
const [stats] = await ds.runQuery(q)
|
|
558
521
|
return stats
|
|
559
522
|
}
|
|
@@ -614,7 +577,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
614
577
|
|
|
615
578
|
override async createTable<ROW extends ObjectWithId>(
|
|
616
579
|
_table: string,
|
|
617
|
-
_schema:
|
|
580
|
+
_schema: JsonSchema<ROW>,
|
|
618
581
|
): Promise<void> {}
|
|
619
582
|
|
|
620
583
|
override async getTables(): Promise<string[]> {
|
|
@@ -623,12 +586,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
623
586
|
return statsArray.map(stats => stats.kind_name).filter(table => table && !table.startsWith('_'))
|
|
624
587
|
}
|
|
625
588
|
|
|
626
|
-
override async getTableSchema<ROW extends ObjectWithId>(
|
|
627
|
-
table: string,
|
|
628
|
-
): Promise<JsonSchemaRootObject<ROW>> {
|
|
589
|
+
override async getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchema<ROW>> {
|
|
629
590
|
const stats = await this.getTableProperties(table)
|
|
630
591
|
|
|
631
|
-
const s:
|
|
592
|
+
const s: JsonSchema<ROW> = {
|
|
632
593
|
$id: `${table}.schema.json`,
|
|
633
594
|
type: 'object',
|
|
634
595
|
properties: {
|
|
@@ -645,40 +606,40 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
645
606
|
const name = stats.property_name as keyof ROW
|
|
646
607
|
|
|
647
608
|
if (dtype === DatastoreType.Blob) {
|
|
648
|
-
s.properties[name] = {
|
|
609
|
+
s.properties![name] = {
|
|
649
610
|
instanceof: 'Buffer',
|
|
650
|
-
} as
|
|
611
|
+
} as JsonSchema<any, ROW[typeof name]>
|
|
651
612
|
} else if (dtype === DatastoreType.Text || dtype === DatastoreType.String) {
|
|
652
|
-
s.properties[name] = {
|
|
613
|
+
s.properties![name] = {
|
|
653
614
|
type: 'string',
|
|
654
|
-
} as
|
|
615
|
+
} as JsonSchema<ROW[typeof name]>
|
|
655
616
|
} else if (dtype === DatastoreType.EmbeddedEntity) {
|
|
656
|
-
s.properties[name] = {
|
|
617
|
+
s.properties![name] = {
|
|
657
618
|
type: 'object',
|
|
658
619
|
additionalProperties: true,
|
|
659
|
-
properties: {},
|
|
620
|
+
properties: {} as any,
|
|
660
621
|
required: [],
|
|
661
|
-
} as
|
|
622
|
+
} as JsonSchema<ROW[typeof name]>
|
|
662
623
|
} else if (dtype === DatastoreType.Integer) {
|
|
663
|
-
s.properties[name] = {
|
|
624
|
+
s.properties![name] = {
|
|
664
625
|
type: 'integer',
|
|
665
|
-
} as
|
|
626
|
+
} as JsonSchema<ROW[typeof name]>
|
|
666
627
|
} else if (dtype === DatastoreType.Float) {
|
|
667
|
-
s.properties[name] = {
|
|
628
|
+
s.properties![name] = {
|
|
668
629
|
type: 'number',
|
|
669
|
-
} as
|
|
630
|
+
} as JsonSchema<ROW[typeof name]>
|
|
670
631
|
} else if (dtype === DatastoreType.Boolean) {
|
|
671
|
-
s.properties[name] = {
|
|
632
|
+
s.properties![name] = {
|
|
672
633
|
type: 'boolean',
|
|
673
|
-
} as
|
|
634
|
+
} as JsonSchema<ROW[typeof name]>
|
|
674
635
|
} else if (dtype === DatastoreType.DATE_TIME) {
|
|
675
636
|
// Don't know how to map it properly
|
|
676
|
-
s.properties[name] = {} as
|
|
637
|
+
s.properties![name] = {} as JsonSchema<any>
|
|
677
638
|
} else if (dtype === DatastoreType.NULL) {
|
|
678
639
|
// check, maybe we can just skip this type and do nothing?
|
|
679
|
-
s.properties[name] ||= {
|
|
640
|
+
s.properties![name] ||= {
|
|
680
641
|
type: 'null',
|
|
681
|
-
} as
|
|
642
|
+
} as JsonSchema<ROW[typeof name]>
|
|
682
643
|
} else {
|
|
683
644
|
throw new Error(
|
|
684
645
|
`Unknown Datastore Type '${stats.property_type}' for ${table}.${name as string}`,
|
package/src/datastore.model.ts
CHANGED
|
@@ -128,12 +128,12 @@ export interface DatastoreStats {
|
|
|
128
128
|
export enum DatastoreType {
|
|
129
129
|
Blob = 'Blob',
|
|
130
130
|
Text = 'Text',
|
|
131
|
-
String = 'String', // eslint-disable-line id-
|
|
131
|
+
String = 'String', // eslint-disable-line id-denylist
|
|
132
132
|
EmbeddedEntity = 'EmbeddedEntity',
|
|
133
133
|
Float = 'Float',
|
|
134
134
|
Integer = 'Integer',
|
|
135
135
|
DATE_TIME = 'Date/Time',
|
|
136
|
-
Boolean = 'Boolean', // eslint-disable-line id-
|
|
136
|
+
Boolean = 'Boolean', // eslint-disable-line id-denylist
|
|
137
137
|
NULL = 'NULL',
|
|
138
138
|
}
|
|
139
139
|
|
package/src/query.util.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { PropertyFilter, type Query } from '@google-cloud/datastore'
|
|
2
2
|
import type { DBQuery, DBQueryFilterOperator } from '@naturalcycles/db-lib'
|
|
3
3
|
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
|
|
4
4
|
|
|
@@ -15,7 +15,6 @@ const OP_MAP: Partial<Record<DBQueryFilterOperator, string>> = {
|
|
|
15
15
|
export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
|
|
16
16
|
dbQuery: Readonly<DBQuery<ROW>>,
|
|
17
17
|
emptyQuery: Query,
|
|
18
|
-
propertyFilterClass: typeof PropertyFilter,
|
|
19
18
|
): Query {
|
|
20
19
|
let q = emptyQuery
|
|
21
20
|
|
|
@@ -31,7 +30,7 @@ export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
|
|
|
31
30
|
// `a == null` will return just that - rows with null values
|
|
32
31
|
let { op, val } = f
|
|
33
32
|
if (val === undefined) val = null
|
|
34
|
-
q = q.filter(new
|
|
33
|
+
q = q.filter(new PropertyFilter(f.name as string, OP_MAP[op] || (op as any), val))
|
|
35
34
|
}
|
|
36
35
|
|
|
37
36
|
// limit
|