@naturalcycles/datastore-lib 3.30.2 → 3.31.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/datastore.db.d.ts +19 -8
- package/dist/datastore.db.js +52 -39
- package/dist/datastore.model.d.ts +1 -3
- package/dist/index.d.ts +3 -5
- package/dist/index.js +4 -7
- package/package.json +2 -2
- package/src/datastore.db.ts +97 -56
- package/src/datastore.model.ts +4 -7
- package/src/index.ts +3 -27
package/dist/datastore.db.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { Transaction } from '@google-cloud/datastore';
|
|
1
2
|
import type { Datastore, Key, Query } from '@google-cloud/datastore';
|
|
2
|
-
import { BaseCommonDB, CommonDB, DBQuery, DBTransaction, RunQueryResult } from '@naturalcycles/db-lib';
|
|
3
|
+
import { BaseCommonDB, CommonDB, CommonDBOptions, CommonDBSaveOptions, CommonDBSupport, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
|
|
3
4
|
import { ObjectWithId, JsonSchemaObject, JsonSchemaRootObject, CommonLogger } from '@naturalcycles/js-lib';
|
|
4
5
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
5
6
|
import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats } from './datastore.model';
|
|
@@ -9,6 +10,7 @@ import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDB
|
|
|
9
10
|
* https://cloud.google.com/datastore/docs/datastore-api-tutorial
|
|
10
11
|
*/
|
|
11
12
|
export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
13
|
+
support: CommonDBSupport;
|
|
12
14
|
constructor(cfg?: DatastoreDBCfg);
|
|
13
15
|
cfg: DatastoreDBCfg & {
|
|
14
16
|
logger: CommonLogger;
|
|
@@ -20,8 +22,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
20
22
|
protected KEY: symbol;
|
|
21
23
|
ds(): Datastore;
|
|
22
24
|
ping(): Promise<void>;
|
|
23
|
-
getByIds<ROW extends ObjectWithId>(table: string, ids:
|
|
24
|
-
getQueryKind(q: Query): string;
|
|
25
|
+
getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<ROW[]>;
|
|
25
26
|
runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<RunQueryResult<ROW>>;
|
|
26
27
|
runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<number>;
|
|
27
28
|
runDatastoreQuery<ROW extends ObjectWithId>(q: Query): Promise<RunQueryResult<ROW>>;
|
|
@@ -36,11 +37,8 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
36
37
|
* Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
|
|
37
38
|
* regardless if they were actually deleted or not.
|
|
38
39
|
*/
|
|
39
|
-
deleteByIds
|
|
40
|
-
|
|
41
|
-
* https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
|
|
42
|
-
*/
|
|
43
|
-
commitTransaction(_tx: DBTransaction, opt?: DatastoreDBSaveOptions): Promise<void>;
|
|
40
|
+
deleteByIds(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<number>;
|
|
41
|
+
runInTransaction(fn: DBTransactionFn): Promise<void>;
|
|
44
42
|
getAllStats(): Promise<DatastoreStats[]>;
|
|
45
43
|
/**
|
|
46
44
|
* Returns undefined e.g when Table is non-existing
|
|
@@ -60,3 +58,16 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
60
58
|
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;
|
|
61
59
|
private getPRetryOptions;
|
|
62
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
|
|
63
|
+
*/
|
|
64
|
+
export declare class DatastoreDBTransaction implements DBTransaction {
|
|
65
|
+
db: DatastoreDB;
|
|
66
|
+
tx: Transaction;
|
|
67
|
+
private constructor();
|
|
68
|
+
static create(db: DatastoreDB): Promise<DatastoreDBTransaction>;
|
|
69
|
+
rollback(): Promise<void>;
|
|
70
|
+
getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: CommonDBOptions | undefined): Promise<ROW[]>;
|
|
71
|
+
saveBatch<ROW extends Partial<ObjectWithId>>(table: string, rows: ROW[], opt?: CommonDBSaveOptions<ROW> | undefined): Promise<void>;
|
|
72
|
+
deleteByIds(table: string, ids: string[], opt?: CommonDBOptions | undefined): Promise<number>;
|
|
73
|
+
}
|
package/dist/datastore.db.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DatastoreDB = void 0;
|
|
3
|
+
exports.DatastoreDBTransaction = exports.DatastoreDB = void 0;
|
|
4
4
|
const node_stream_1 = require("node:stream");
|
|
5
5
|
const datastore_1 = require("@google-cloud/datastore");
|
|
6
6
|
const db_lib_1 = require("@naturalcycles/db-lib");
|
|
@@ -35,6 +35,10 @@ const methodMap = {
|
|
|
35
35
|
class DatastoreDB extends db_lib_1.BaseCommonDB {
|
|
36
36
|
constructor(cfg = {}) {
|
|
37
37
|
super();
|
|
38
|
+
this.support = {
|
|
39
|
+
...db_lib_1.commonDBFullSupport,
|
|
40
|
+
updateByQuery: false,
|
|
41
|
+
};
|
|
38
42
|
this.cfg = {
|
|
39
43
|
logger: console,
|
|
40
44
|
...cfg,
|
|
@@ -65,7 +69,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
|
|
|
65
69
|
async ping() {
|
|
66
70
|
await this.getAllStats();
|
|
67
71
|
}
|
|
68
|
-
async getByIds(table, ids,
|
|
72
|
+
async getByIds(table, ids, opt = {}) {
|
|
69
73
|
if (!ids.length)
|
|
70
74
|
return [];
|
|
71
75
|
const keys = ids.map(id => this.key(table, id));
|
|
@@ -73,7 +77,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
|
|
|
73
77
|
if (this.cfg.timeout) {
|
|
74
78
|
// First try
|
|
75
79
|
try {
|
|
76
|
-
const r = await (0, js_lib_1.pTimeout)(() => this.ds().get(keys), {
|
|
80
|
+
const r = await (0, js_lib_1.pTimeout)(() => (opt.tx?.tx || this.ds()).get(keys), {
|
|
77
81
|
timeout: this.cfg.timeout,
|
|
78
82
|
name: `datastore.getByIds(${table})`,
|
|
79
83
|
});
|
|
@@ -86,7 +90,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
|
|
|
86
90
|
const DS = datastoreLib.Datastore;
|
|
87
91
|
this.cachedDatastore = new DS(this.cfg);
|
|
88
92
|
// Second try (will throw)
|
|
89
|
-
const r = await (0, js_lib_1.pRetry)(() => this.ds().get(keys), {
|
|
93
|
+
const r = await (0, js_lib_1.pRetry)(() => (opt.tx?.tx || this.ds()).get(keys), {
|
|
90
94
|
...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
|
|
91
95
|
maxAttempts: 3,
|
|
92
96
|
timeout: this.cfg.timeout,
|
|
@@ -109,11 +113,10 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
|
|
|
109
113
|
// same ids are not expected here
|
|
110
114
|
.sort((a, b) => (a.id > b.id ? 1 : -1)));
|
|
111
115
|
}
|
|
112
|
-
getQueryKind(q) {
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
}
|
|
116
|
+
// getQueryKind(q: Query): string {
|
|
117
|
+
// if (!q?.kinds?.length) return '' // should never be the case, but
|
|
118
|
+
// return q.kinds[0]!
|
|
119
|
+
// }
|
|
117
120
|
async runQuery(dbQuery, _opt) {
|
|
118
121
|
const idFilter = dbQuery._filters.find(f => f.name === 'id');
|
|
119
122
|
if (idFilter) {
|
|
@@ -172,7 +175,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
|
|
|
172
175
|
const entities = rows.map(obj => this.toDatastoreEntity(table, obj, opt.excludeFromIndexes));
|
|
173
176
|
const method = methodMap[opt.saveMethod || 'upsert'] || 'save';
|
|
174
177
|
const save = (0, js_lib_1.pRetryFn)(async (batch) => {
|
|
175
|
-
await (opt.tx || this.ds())[method](batch);
|
|
178
|
+
await (opt.tx?.tx || this.ds())[method](batch);
|
|
176
179
|
}, this.getPRetryOptions(`DatastoreLib.saveBatch(${table})`));
|
|
177
180
|
try {
|
|
178
181
|
const chunks = (0, js_lib_1._chunk)(entities, MAX_ITEMS);
|
|
@@ -206,38 +209,21 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
|
|
|
206
209
|
*/
|
|
207
210
|
async deleteByIds(table, ids, opt = {}) {
|
|
208
211
|
const keys = ids.map(id => this.key(table, id));
|
|
212
|
+
await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(keys, MAX_ITEMS),
|
|
209
213
|
// eslint-disable-next-line @typescript-eslint/return-await
|
|
210
|
-
|
|
214
|
+
async (batch) => await (opt.tx?.tx || this.ds()).delete(batch));
|
|
211
215
|
return ids.length;
|
|
212
216
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
for await (const op of ops) {
|
|
224
|
-
if (op.type === 'saveBatch') {
|
|
225
|
-
await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt, tx });
|
|
226
|
-
}
|
|
227
|
-
else if (op.type === 'deleteByIds') {
|
|
228
|
-
await this.deleteByIds(op.table, op.ids, { ...op.opt, ...opt, tx });
|
|
229
|
-
}
|
|
230
|
-
else {
|
|
231
|
-
throw new Error(`DBOperation not supported: ${op.type}`);
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
await tx.commit();
|
|
235
|
-
}
|
|
236
|
-
catch (err) {
|
|
237
|
-
await tx.rollback();
|
|
238
|
-
throw err; // rethrow
|
|
239
|
-
}
|
|
240
|
-
}, this.getPRetryOptions(`DatastoreLib.commitTransaction`));
|
|
217
|
+
async runInTransaction(fn) {
|
|
218
|
+
const tx = await DatastoreDBTransaction.create(this);
|
|
219
|
+
try {
|
|
220
|
+
await fn(tx);
|
|
221
|
+
await tx.tx.commit();
|
|
222
|
+
}
|
|
223
|
+
catch (err) {
|
|
224
|
+
await tx.tx.rollback();
|
|
225
|
+
throw err;
|
|
226
|
+
}
|
|
241
227
|
}
|
|
242
228
|
async getAllStats() {
|
|
243
229
|
const q = this.ds().createQuery('__Stat_Kind__');
|
|
@@ -393,3 +379,30 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
|
|
|
393
379
|
}
|
|
394
380
|
}
|
|
395
381
|
exports.DatastoreDB = DatastoreDB;
|
|
382
|
+
/**
|
|
383
|
+
* https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
|
|
384
|
+
*/
|
|
385
|
+
class DatastoreDBTransaction {
|
|
386
|
+
constructor(db, tx) {
|
|
387
|
+
this.db = db;
|
|
388
|
+
this.tx = tx;
|
|
389
|
+
}
|
|
390
|
+
static async create(db) {
|
|
391
|
+
const tx = db.ds().transaction();
|
|
392
|
+
await tx.run();
|
|
393
|
+
return new DatastoreDBTransaction(db, tx);
|
|
394
|
+
}
|
|
395
|
+
async rollback() {
|
|
396
|
+
await this.tx.rollback();
|
|
397
|
+
}
|
|
398
|
+
async getByIds(table, ids, opt) {
|
|
399
|
+
return await this.db.getByIds(table, ids, { ...opt, tx: this });
|
|
400
|
+
}
|
|
401
|
+
async saveBatch(table, rows, opt) {
|
|
402
|
+
await this.db.saveBatch(table, rows, { ...opt, tx: this });
|
|
403
|
+
}
|
|
404
|
+
async deleteByIds(table, ids, opt) {
|
|
405
|
+
return await this.db.deleteByIds(table, ids, { ...opt, tx: this });
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
exports.DatastoreDBTransaction = DatastoreDBTransaction;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DatastoreOptions, Key
|
|
1
|
+
import type { DatastoreOptions, Key } from '@google-cloud/datastore';
|
|
2
2
|
import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib';
|
|
3
3
|
import { CommonLogger, NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib';
|
|
4
4
|
export interface DatastorePayload<T = any> {
|
|
@@ -100,10 +100,8 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
|
|
|
100
100
|
maxWait?: NumberOfSeconds;
|
|
101
101
|
}
|
|
102
102
|
export interface DatastoreDBOptions extends CommonDBOptions {
|
|
103
|
-
tx?: Transaction;
|
|
104
103
|
}
|
|
105
104
|
export interface DatastoreDBSaveOptions<ROW extends Partial<ObjectWithId> = any> extends CommonDBSaveOptions<ROW> {
|
|
106
|
-
tx?: Transaction;
|
|
107
105
|
}
|
|
108
106
|
export interface DatastoreStats {
|
|
109
107
|
composite_index_count: number;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
export type { DatastorePayload, DatastoreDBCfg, DatastoreKeyValueDBCfg, DatastoreStats, DatastoreCredentials, DatastorePropertyStats, DatastoreDBStreamOptions, DatastoreDBOptions, DatastoreDBSaveOptions, };
|
|
5
|
-
export { DatastoreDB, DatastoreType, DatastoreKeyValueDB };
|
|
1
|
+
export * from './datastore.db';
|
|
2
|
+
export * from './datastore.model';
|
|
3
|
+
export * from './datastoreKeyValueDB';
|
package/dist/index.js
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
Object.defineProperty(exports, "DatastoreType", { enumerable: true, get: function () { return datastore_model_1.DatastoreType; } });
|
|
8
|
-
const datastoreKeyValueDB_1 = require("./datastoreKeyValueDB");
|
|
9
|
-
Object.defineProperty(exports, "DatastoreKeyValueDB", { enumerable: true, get: function () { return datastoreKeyValueDB_1.DatastoreKeyValueDB; } });
|
|
3
|
+
const tslib_1 = require("tslib");
|
|
4
|
+
tslib_1.__exportStar(require("./datastore.db"), exports);
|
|
5
|
+
tslib_1.__exportStar(require("./datastore.model"), exports);
|
|
6
|
+
tslib_1.__exportStar(require("./datastoreKeyValueDB"), exports);
|
package/package.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/datastore-lib",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.31.1",
|
|
4
4
|
"description": "Opinionated library to work with Google Datastore",
|
|
5
5
|
"scripts": {
|
|
6
6
|
"prepare": "husky install"
|
|
7
7
|
},
|
|
8
8
|
"dependencies": {
|
|
9
9
|
"@google-cloud/datastore": "^8.0.0",
|
|
10
|
-
"@naturalcycles/db-lib": "^
|
|
10
|
+
"@naturalcycles/db-lib": "^9.0.0",
|
|
11
11
|
"@naturalcycles/js-lib": "^14.116.0",
|
|
12
12
|
"@naturalcycles/nodejs-lib": "^13.1.0"
|
|
13
13
|
},
|
package/src/datastore.db.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { Transform } from 'node:stream'
|
|
2
|
-
import { PropertyFilter } from '@google-cloud/datastore'
|
|
2
|
+
import { PropertyFilter, Transaction } from '@google-cloud/datastore'
|
|
3
3
|
import type { Datastore, Key, Query } from '@google-cloud/datastore'
|
|
4
4
|
import {
|
|
5
5
|
BaseCommonDB,
|
|
6
6
|
CommonDB,
|
|
7
|
+
commonDBFullSupport,
|
|
8
|
+
CommonDBOptions,
|
|
7
9
|
CommonDBSaveMethod,
|
|
10
|
+
CommonDBSaveOptions,
|
|
11
|
+
CommonDBSupport,
|
|
8
12
|
DBQuery,
|
|
9
13
|
DBTransaction,
|
|
10
|
-
|
|
14
|
+
DBTransactionFn,
|
|
11
15
|
RunQueryResult,
|
|
12
16
|
} from '@naturalcycles/db-lib'
|
|
13
17
|
import {
|
|
@@ -72,6 +76,11 @@ const methodMap: Record<CommonDBSaveMethod, string> = {
|
|
|
72
76
|
* https://cloud.google.com/datastore/docs/datastore-api-tutorial
|
|
73
77
|
*/
|
|
74
78
|
export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
79
|
+
override support: CommonDBSupport = {
|
|
80
|
+
...commonDBFullSupport,
|
|
81
|
+
updateByQuery: false,
|
|
82
|
+
}
|
|
83
|
+
|
|
75
84
|
constructor(cfg: DatastoreDBCfg = {}) {
|
|
76
85
|
super()
|
|
77
86
|
this.cfg = {
|
|
@@ -125,8 +134,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
125
134
|
|
|
126
135
|
override async getByIds<ROW extends ObjectWithId>(
|
|
127
136
|
table: string,
|
|
128
|
-
ids:
|
|
129
|
-
|
|
137
|
+
ids: string[],
|
|
138
|
+
opt: DatastoreDBOptions = {},
|
|
130
139
|
): Promise<ROW[]> {
|
|
131
140
|
if (!ids.length) return []
|
|
132
141
|
const keys = ids.map(id => this.key(table, id))
|
|
@@ -135,10 +144,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
135
144
|
if (this.cfg.timeout) {
|
|
136
145
|
// First try
|
|
137
146
|
try {
|
|
138
|
-
const r = await pTimeout(
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
147
|
+
const r = await pTimeout(
|
|
148
|
+
() => ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).get(keys),
|
|
149
|
+
{
|
|
150
|
+
timeout: this.cfg.timeout,
|
|
151
|
+
name: `datastore.getByIds(${table})`,
|
|
152
|
+
},
|
|
153
|
+
)
|
|
142
154
|
rows = r[0]
|
|
143
155
|
} catch {
|
|
144
156
|
this.cfg.logger.log('datastore recreated on error')
|
|
@@ -149,15 +161,18 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
149
161
|
this.cachedDatastore = new DS(this.cfg)
|
|
150
162
|
|
|
151
163
|
// Second try (will throw)
|
|
152
|
-
const r = await pRetry(
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
164
|
+
const r = await pRetry(
|
|
165
|
+
() => ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).get(keys),
|
|
166
|
+
{
|
|
167
|
+
...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
|
|
168
|
+
maxAttempts: 3,
|
|
169
|
+
timeout: this.cfg.timeout,
|
|
170
|
+
errorData: {
|
|
171
|
+
// This error will be grouped ACROSS all endpoints and usages
|
|
172
|
+
fingerprint: [DATASTORE_TIMEOUT],
|
|
173
|
+
},
|
|
159
174
|
},
|
|
160
|
-
|
|
175
|
+
)
|
|
161
176
|
rows = r[0]
|
|
162
177
|
}
|
|
163
178
|
} else {
|
|
@@ -178,10 +193,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
178
193
|
)
|
|
179
194
|
}
|
|
180
195
|
|
|
181
|
-
getQueryKind(q: Query): string {
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
}
|
|
196
|
+
// getQueryKind(q: Query): string {
|
|
197
|
+
// if (!q?.kinds?.length) return '' // should never be the case, but
|
|
198
|
+
// return q.kinds[0]!
|
|
199
|
+
// }
|
|
185
200
|
|
|
186
201
|
override async runQuery<ROW extends ObjectWithId>(
|
|
187
202
|
dbQuery: DBQuery<ROW>,
|
|
@@ -284,7 +299,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
284
299
|
|
|
285
300
|
const save = pRetryFn(
|
|
286
301
|
async (batch: DatastorePayload<ROW>[]) => {
|
|
287
|
-
await (opt.tx || this.ds())[method](batch)
|
|
302
|
+
await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds())[method](batch)
|
|
288
303
|
},
|
|
289
304
|
this.getPRetryOptions(`DatastoreLib.saveBatch(${table})`),
|
|
290
305
|
)
|
|
@@ -331,49 +346,31 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
331
346
|
* Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
|
|
332
347
|
* regardless if they were actually deleted or not.
|
|
333
348
|
*/
|
|
334
|
-
async deleteByIds
|
|
349
|
+
override async deleteByIds(
|
|
335
350
|
table: string,
|
|
336
|
-
ids:
|
|
351
|
+
ids: string[],
|
|
337
352
|
opt: DatastoreDBOptions = {},
|
|
338
353
|
): Promise<number> {
|
|
339
354
|
const keys = ids.map(id => this.key(table, id))
|
|
340
|
-
|
|
341
|
-
await pMap(
|
|
355
|
+
|
|
356
|
+
await pMap(
|
|
357
|
+
_chunk(keys, MAX_ITEMS),
|
|
358
|
+
// eslint-disable-next-line @typescript-eslint/return-await
|
|
359
|
+
async batch => await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).delete(batch),
|
|
360
|
+
)
|
|
342
361
|
return ids.length
|
|
343
362
|
}
|
|
344
363
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
*/
|
|
348
|
-
override async commitTransaction(
|
|
349
|
-
_tx: DBTransaction,
|
|
350
|
-
opt?: DatastoreDBSaveOptions,
|
|
351
|
-
): Promise<void> {
|
|
352
|
-
// Using Retry, because Datastore can throw errors like "too much contention" here
|
|
353
|
-
await pRetry(async () => {
|
|
354
|
-
const tx = this.ds().transaction()
|
|
355
|
-
|
|
356
|
-
try {
|
|
357
|
-
await tx.run()
|
|
358
|
-
|
|
359
|
-
const ops = mergeDBOperations(_tx.ops)
|
|
360
|
-
|
|
361
|
-
for await (const op of ops) {
|
|
362
|
-
if (op.type === 'saveBatch') {
|
|
363
|
-
await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt, tx })
|
|
364
|
-
} else if (op.type === 'deleteByIds') {
|
|
365
|
-
await this.deleteByIds(op.table, op.ids, { ...op.opt, ...opt, tx })
|
|
366
|
-
} else {
|
|
367
|
-
throw new Error(`DBOperation not supported: ${(op as any).type}`)
|
|
368
|
-
}
|
|
369
|
-
}
|
|
364
|
+
override async runInTransaction(fn: DBTransactionFn): Promise<void> {
|
|
365
|
+
const tx = await DatastoreDBTransaction.create(this)
|
|
370
366
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
367
|
+
try {
|
|
368
|
+
await fn(tx)
|
|
369
|
+
await tx.tx.commit()
|
|
370
|
+
} catch (err) {
|
|
371
|
+
await tx.tx.rollback()
|
|
372
|
+
throw err
|
|
373
|
+
}
|
|
377
374
|
}
|
|
378
375
|
|
|
379
376
|
async getAllStats(): Promise<DatastoreStats[]> {
|
|
@@ -547,3 +544,47 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
|
|
|
547
544
|
}
|
|
548
545
|
}
|
|
549
546
|
}
|
|
547
|
+
|
|
548
|
+
/**
|
|
549
|
+
* https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
|
|
550
|
+
*/
|
|
551
|
+
export class DatastoreDBTransaction implements DBTransaction {
|
|
552
|
+
private constructor(
|
|
553
|
+
public db: DatastoreDB,
|
|
554
|
+
public tx: Transaction,
|
|
555
|
+
) {}
|
|
556
|
+
|
|
557
|
+
static async create(db: DatastoreDB): Promise<DatastoreDBTransaction> {
|
|
558
|
+
const tx = db.ds().transaction()
|
|
559
|
+
await tx.run()
|
|
560
|
+
return new DatastoreDBTransaction(db, tx)
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
async rollback(): Promise<void> {
|
|
564
|
+
await this.tx.rollback()
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
async getByIds<ROW extends ObjectWithId>(
|
|
568
|
+
table: string,
|
|
569
|
+
ids: string[],
|
|
570
|
+
opt?: CommonDBOptions | undefined,
|
|
571
|
+
): Promise<ROW[]> {
|
|
572
|
+
return await this.db.getByIds(table, ids, { ...opt, tx: this })
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
async saveBatch<ROW extends Partial<ObjectWithId>>(
|
|
576
|
+
table: string,
|
|
577
|
+
rows: ROW[],
|
|
578
|
+
opt?: CommonDBSaveOptions<ROW> | undefined,
|
|
579
|
+
): Promise<void> {
|
|
580
|
+
await this.db.saveBatch(table, rows, { ...opt, tx: this })
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
async deleteByIds(
|
|
584
|
+
table: string,
|
|
585
|
+
ids: string[],
|
|
586
|
+
opt?: CommonDBOptions | undefined,
|
|
587
|
+
): Promise<number> {
|
|
588
|
+
return await this.db.deleteByIds(table, ids, { ...opt, tx: this })
|
|
589
|
+
}
|
|
590
|
+
}
|
package/src/datastore.model.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { DatastoreOptions, Key
|
|
1
|
+
import type { DatastoreOptions, Key } from '@google-cloud/datastore'
|
|
2
2
|
import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib'
|
|
3
3
|
import { CommonLogger, NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib'
|
|
4
4
|
|
|
@@ -114,13 +114,10 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
|
|
|
114
114
|
maxWait?: NumberOfSeconds
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
export interface DatastoreDBOptions extends CommonDBOptions {
|
|
118
|
-
|
|
119
|
-
}
|
|
117
|
+
export interface DatastoreDBOptions extends CommonDBOptions {}
|
|
118
|
+
|
|
120
119
|
export interface DatastoreDBSaveOptions<ROW extends Partial<ObjectWithId> = any>
|
|
121
|
-
extends CommonDBSaveOptions<ROW> {
|
|
122
|
-
tx?: Transaction
|
|
123
|
-
}
|
|
120
|
+
extends CommonDBSaveOptions<ROW> {}
|
|
124
121
|
|
|
125
122
|
export interface DatastoreStats {
|
|
126
123
|
composite_index_count: number
|
package/src/index.ts
CHANGED
|
@@ -1,27 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
DatastoreDBCfg,
|
|
5
|
-
DatastoreDBOptions,
|
|
6
|
-
DatastoreDBSaveOptions,
|
|
7
|
-
DatastoreDBStreamOptions,
|
|
8
|
-
DatastorePayload,
|
|
9
|
-
DatastorePropertyStats,
|
|
10
|
-
DatastoreStats,
|
|
11
|
-
DatastoreType,
|
|
12
|
-
} from './datastore.model'
|
|
13
|
-
import { DatastoreKeyValueDB, DatastoreKeyValueDBCfg } from './datastoreKeyValueDB'
|
|
14
|
-
|
|
15
|
-
export type {
|
|
16
|
-
DatastorePayload,
|
|
17
|
-
DatastoreDBCfg,
|
|
18
|
-
DatastoreKeyValueDBCfg,
|
|
19
|
-
DatastoreStats,
|
|
20
|
-
DatastoreCredentials,
|
|
21
|
-
DatastorePropertyStats,
|
|
22
|
-
DatastoreDBStreamOptions,
|
|
23
|
-
DatastoreDBOptions,
|
|
24
|
-
DatastoreDBSaveOptions,
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export { DatastoreDB, DatastoreType, DatastoreKeyValueDB }
|
|
1
|
+
export * from './datastore.db'
|
|
2
|
+
export * from './datastore.model'
|
|
3
|
+
export * from './datastoreKeyValueDB'
|