@naturalcycles/db-lib 8.60.1 → 9.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/cachedb/cache.db.d.ts +3 -4
- package/dist/adapter/cachedb/cache.db.js +5 -4
- package/dist/adapter/cachedb/cache.db.model.d.ts +2 -2
- package/dist/adapter/file/file.db.d.ts +4 -7
- package/dist/adapter/file/file.db.js +102 -56
- package/dist/adapter/inmemory/inMemory.db.d.ts +32 -3
- package/dist/adapter/inmemory/inMemory.db.js +76 -25
- package/dist/base.common.db.d.ts +7 -10
- package/dist/base.common.db.js +13 -7
- package/dist/common.db.d.ts +60 -4
- package/dist/common.db.js +23 -0
- package/dist/commondao/common.dao.d.ts +28 -9
- package/dist/commondao/common.dao.js +72 -71
- package/dist/commondao/common.dao.model.d.ts +0 -10
- package/dist/db.model.d.ts +29 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/testing/daoTest.d.ts +2 -2
- package/dist/testing/daoTest.js +29 -39
- package/dist/testing/dbTest.d.ts +1 -39
- package/dist/testing/dbTest.js +41 -50
- package/dist/testing/index.d.ts +2 -2
- package/dist/timeseries/commonTimeSeriesDao.js +9 -10
- package/dist/transaction/dbTransaction.util.d.ts +14 -4
- package/dist/transaction/dbTransaction.util.js +49 -22
- package/dist/validation/index.js +2 -2
- package/package.json +1 -1
- package/src/adapter/cachedb/cache.db.model.ts +7 -2
- package/src/adapter/cachedb/cache.db.ts +7 -8
- package/src/adapter/file/file.db.ts +120 -74
- package/src/adapter/inmemory/inMemory.db.ts +101 -24
- package/src/base.common.db.ts +22 -11
- package/src/common.db.ts +84 -3
- package/src/commondao/common.dao.model.ts +0 -11
- package/src/commondao/common.dao.ts +102 -91
- package/src/db.model.ts +34 -2
- package/src/index.ts +0 -1
- package/src/testing/daoTest.ts +32 -52
- package/src/testing/dbTest.ts +43 -120
- package/src/testing/index.ts +2 -12
- package/src/timeseries/commonTimeSeriesDao.ts +9 -12
- package/src/transaction/dbTransaction.util.ts +61 -23
- package/src/validation/index.ts +2 -2
- package/dist/transaction/dbTransaction.d.ts +0 -27
- package/dist/transaction/dbTransaction.js +0 -64
- package/src/transaction/dbTransaction.ts +0 -67
|
@@ -2,10 +2,9 @@
|
|
|
2
2
|
import { Readable } from 'node:stream';
|
|
3
3
|
import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
|
|
4
4
|
import { BaseCommonDB } from '../../base.common.db';
|
|
5
|
-
import { CommonDB } from '../../common.db';
|
|
6
|
-
import {
|
|
5
|
+
import { CommonDB, CommonDBSupport } from '../../common.db';
|
|
6
|
+
import { DBPatch, RunQueryResult } from '../../db.model';
|
|
7
7
|
import { DBQuery } from '../../query/dbQuery';
|
|
8
|
-
import { DBTransaction } from '../../transaction/dbTransaction';
|
|
9
8
|
import { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBSaveOptions, CacheDBStreamOptions } from './cache.db.model';
|
|
10
9
|
/**
|
|
11
10
|
* CommonDB implementation that proxies requests to downstream CommonDB
|
|
@@ -14,6 +13,7 @@ import { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBSaveOptions, C
|
|
|
14
13
|
* Queries always hit downstream (unless `onlyCache` is passed)
|
|
15
14
|
*/
|
|
16
15
|
export declare class CacheDB extends BaseCommonDB implements CommonDB {
|
|
16
|
+
support: CommonDBSupport;
|
|
17
17
|
constructor(cfg: CacheDBCfg);
|
|
18
18
|
cfg: CacheDBCfg;
|
|
19
19
|
ping(): Promise<void>;
|
|
@@ -30,5 +30,4 @@ export declare class CacheDB extends BaseCommonDB implements CommonDB {
|
|
|
30
30
|
streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CacheDBStreamOptions): Readable;
|
|
31
31
|
deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CacheDBOptions): Promise<number>;
|
|
32
32
|
updateByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, patch: DBPatch<ROW>, opt?: CacheDBOptions): Promise<number>;
|
|
33
|
-
commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void>;
|
|
34
33
|
}
|
|
@@ -4,6 +4,7 @@ exports.CacheDB = void 0;
|
|
|
4
4
|
const node_stream_1 = require("node:stream");
|
|
5
5
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
6
|
const base_common_db_1 = require("../../base.common.db");
|
|
7
|
+
const common_db_1 = require("../../common.db");
|
|
7
8
|
/**
|
|
8
9
|
* CommonDB implementation that proxies requests to downstream CommonDB
|
|
9
10
|
* and does in-memory caching.
|
|
@@ -13,6 +14,10 @@ const base_common_db_1 = require("../../base.common.db");
|
|
|
13
14
|
class CacheDB extends base_common_db_1.BaseCommonDB {
|
|
14
15
|
constructor(cfg) {
|
|
15
16
|
super();
|
|
17
|
+
this.support = {
|
|
18
|
+
...common_db_1.commonDBFullSupport,
|
|
19
|
+
transactions: false,
|
|
20
|
+
};
|
|
16
21
|
this.cfg = {
|
|
17
22
|
logger: console,
|
|
18
23
|
...cfg,
|
|
@@ -187,9 +192,5 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
|
|
|
187
192
|
}
|
|
188
193
|
return updated || 0;
|
|
189
194
|
}
|
|
190
|
-
async commitTransaction(tx, opt) {
|
|
191
|
-
await this.cfg.downstreamDB.commitTransaction(tx, opt);
|
|
192
|
-
await this.cfg.cacheDB.commitTransaction(tx, opt);
|
|
193
|
-
}
|
|
194
195
|
}
|
|
195
196
|
exports.CacheDB = CacheDB;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
2
|
import { CommonDB } from '../../common.db';
|
|
3
|
-
import { CommonDBCreateOptions, CommonDBSaveOptions, CommonDBStreamOptions } from '../../db.model';
|
|
3
|
+
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions } from '../../db.model';
|
|
4
4
|
export interface CacheDBCfg {
|
|
5
5
|
name: string;
|
|
6
6
|
cacheDB: CommonDB;
|
|
@@ -37,7 +37,7 @@ export interface CacheDBCfg {
|
|
|
37
37
|
*/
|
|
38
38
|
logger?: CommonLogger;
|
|
39
39
|
}
|
|
40
|
-
export interface CacheDBOptions {
|
|
40
|
+
export interface CacheDBOptions extends CommonDBOptions {
|
|
41
41
|
/**
|
|
42
42
|
* @default false
|
|
43
43
|
*/
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
-
import { BaseCommonDB, DBSaveBatchOperation } from '../..';
|
|
3
|
+
import { BaseCommonDB, CommonDBSupport, DBSaveBatchOperation } from '../..';
|
|
4
4
|
import { CommonDB } from '../../common.db';
|
|
5
5
|
import { CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, RunQueryResult } from '../../db.model';
|
|
6
6
|
import { DBQuery } from '../../query/dbQuery';
|
|
7
|
-
import { DBTransaction } from '../../transaction/dbTransaction';
|
|
8
7
|
import { FileDBCfg } from './file.db.model';
|
|
9
8
|
/**
|
|
10
9
|
* Provides barebone implementation for "whole file" based CommonDB.
|
|
@@ -17,25 +16,23 @@ import { FileDBCfg } from './file.db.model';
|
|
|
17
16
|
* Each save operation saves *whole* file to the persistence layer.
|
|
18
17
|
*/
|
|
19
18
|
export declare class FileDB extends BaseCommonDB implements CommonDB {
|
|
19
|
+
support: CommonDBSupport;
|
|
20
20
|
constructor(cfg: FileDBCfg);
|
|
21
21
|
cfg: FileDBCfg;
|
|
22
22
|
ping(): Promise<void>;
|
|
23
23
|
getTables(): Promise<string[]>;
|
|
24
24
|
getByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][], _opt?: CommonDBOptions): Promise<ROW[]>;
|
|
25
25
|
saveBatch<ROW extends Partial<ObjectWithId>>(table: string, rows: ROW[], _opt?: CommonDBSaveOptions<ROW>): Promise<void>;
|
|
26
|
-
/**
|
|
27
|
-
* Implementation is optimized for loading/saving _whole files_.
|
|
28
|
-
*/
|
|
29
|
-
commitTransaction(tx: DBTransaction, _opt?: CommonDBOptions): Promise<void>;
|
|
30
26
|
runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<RunQueryResult<ROW>>;
|
|
31
27
|
runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
|
|
32
28
|
streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBStreamOptions): ReadableTyped<ROW>;
|
|
33
29
|
deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
|
|
30
|
+
deleteByIds(table: string, ids: string[], _opt?: CommonDBOptions): Promise<number>;
|
|
34
31
|
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;
|
|
35
32
|
loadFile<ROW extends ObjectWithId>(table: string): Promise<ROW[]>;
|
|
36
33
|
saveFile<ROW extends ObjectWithId>(table: string, _rows: ROW[]): Promise<void>;
|
|
37
34
|
saveFiles<ROW extends ObjectWithId>(ops: DBSaveBatchOperation<ROW>[]): Promise<void>;
|
|
38
|
-
|
|
35
|
+
sortRows<ROW extends ObjectWithId>(rows: ROW[]): ROW[];
|
|
39
36
|
private logStarted;
|
|
40
37
|
private logFinished;
|
|
41
38
|
}
|
|
@@ -17,6 +17,15 @@ const __1 = require("../..");
|
|
|
17
17
|
class FileDB extends __1.BaseCommonDB {
|
|
18
18
|
constructor(cfg) {
|
|
19
19
|
super();
|
|
20
|
+
this.support = {
|
|
21
|
+
...__1.commonDBFullSupport,
|
|
22
|
+
bufferValues: false, // todo: implement
|
|
23
|
+
insertSaveMethod: false,
|
|
24
|
+
updateSaveMethod: false,
|
|
25
|
+
updateByQuery: false,
|
|
26
|
+
createTable: false,
|
|
27
|
+
transactions: false, // todo
|
|
28
|
+
};
|
|
20
29
|
this.cfg = {
|
|
21
30
|
sortObjects: true,
|
|
22
31
|
logFinished: true,
|
|
@@ -57,62 +66,6 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
57
66
|
await this.saveFile(table, (0, js_lib_1._stringMapValues)(byId));
|
|
58
67
|
}
|
|
59
68
|
}
|
|
60
|
-
/**
|
|
61
|
-
* Implementation is optimized for loading/saving _whole files_.
|
|
62
|
-
*/
|
|
63
|
-
async commitTransaction(tx, _opt) {
|
|
64
|
-
// data[table][id] => row
|
|
65
|
-
const data = {};
|
|
66
|
-
// 1. Load all tables data (concurrently)
|
|
67
|
-
const tables = (0, js_lib_1._uniq)(tx.ops.map(o => o.table));
|
|
68
|
-
await (0, js_lib_1.pMap)(tables, async (table) => {
|
|
69
|
-
const rows = await this.loadFile(table);
|
|
70
|
-
data[table] = (0, js_lib_1._by)(rows, r => r.id);
|
|
71
|
-
}, { concurrency: 16 });
|
|
72
|
-
const backup = (0, js_lib_1._deepCopy)(data);
|
|
73
|
-
// 2. Apply ops one by one (in order)
|
|
74
|
-
tx.ops.forEach(op => {
|
|
75
|
-
if (op.type === 'deleteByIds') {
|
|
76
|
-
op.ids.forEach(id => delete data[op.table][id]);
|
|
77
|
-
}
|
|
78
|
-
else if (op.type === 'saveBatch') {
|
|
79
|
-
op.rows.forEach(r => {
|
|
80
|
-
if (!r.id) {
|
|
81
|
-
throw new Error('FileDB: row has an empty id');
|
|
82
|
-
}
|
|
83
|
-
data[op.table][r.id] = r;
|
|
84
|
-
});
|
|
85
|
-
}
|
|
86
|
-
else {
|
|
87
|
-
throw new Error(`DBOperation not supported: ${op.type}`);
|
|
88
|
-
}
|
|
89
|
-
});
|
|
90
|
-
// 3. Sort, turn it into ops
|
|
91
|
-
// Not filtering empty arrays, cause it's already filtered in this.saveFiles()
|
|
92
|
-
const ops = (0, js_lib_1._stringMapEntries)(data).map(([table, map]) => {
|
|
93
|
-
return {
|
|
94
|
-
type: 'saveBatch',
|
|
95
|
-
table,
|
|
96
|
-
rows: this.sortRows((0, js_lib_1._stringMapValues)(map)),
|
|
97
|
-
};
|
|
98
|
-
});
|
|
99
|
-
// 4. Save all files
|
|
100
|
-
try {
|
|
101
|
-
await this.saveFiles(ops);
|
|
102
|
-
}
|
|
103
|
-
catch (err) {
|
|
104
|
-
const ops = (0, js_lib_1._stringMapEntries)(backup).map(([table, map]) => {
|
|
105
|
-
return {
|
|
106
|
-
type: 'saveBatch',
|
|
107
|
-
table,
|
|
108
|
-
rows: this.sortRows((0, js_lib_1._stringMapValues)(map)),
|
|
109
|
-
};
|
|
110
|
-
});
|
|
111
|
-
// Rollback, ignore rollback error (if any)
|
|
112
|
-
await this.saveFiles(ops).catch(_ => { });
|
|
113
|
-
throw err;
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
69
|
async runQuery(q, _opt) {
|
|
117
70
|
return {
|
|
118
71
|
rows: (0, __1.queryInMemory)(q, await this.loadFile(q.table)),
|
|
@@ -141,6 +94,20 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
141
94
|
}
|
|
142
95
|
return deleted;
|
|
143
96
|
}
|
|
97
|
+
async deleteByIds(table, ids, _opt) {
|
|
98
|
+
const byId = (0, js_lib_1._by)(await this.loadFile(table), r => r.id);
|
|
99
|
+
let deleted = 0;
|
|
100
|
+
ids.forEach(id => {
|
|
101
|
+
if (!byId[id])
|
|
102
|
+
return;
|
|
103
|
+
delete byId[id];
|
|
104
|
+
deleted++;
|
|
105
|
+
});
|
|
106
|
+
if (deleted > 0) {
|
|
107
|
+
await this.saveFile(table, (0, js_lib_1._stringMapValues)(byId));
|
|
108
|
+
}
|
|
109
|
+
return deleted;
|
|
110
|
+
}
|
|
144
111
|
async getTableSchema(table) {
|
|
145
112
|
const rows = await this.loadFile(table);
|
|
146
113
|
return {
|
|
@@ -173,6 +140,9 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
173
140
|
await this.cfg.plugin.saveFiles(ops);
|
|
174
141
|
this.logFinished(started, op);
|
|
175
142
|
}
|
|
143
|
+
// override async createTransaction(): Promise<FileDBTransaction> {
|
|
144
|
+
// return new FileDBTransaction(this)
|
|
145
|
+
// }
|
|
176
146
|
sortRows(rows) {
|
|
177
147
|
rows = rows.map(r => (0, js_lib_1._filterUndefinedValues)(r));
|
|
178
148
|
if (this.cfg.sortOnSave) {
|
|
@@ -198,3 +168,79 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
198
168
|
}
|
|
199
169
|
}
|
|
200
170
|
exports.FileDB = FileDB;
|
|
171
|
+
// todo: get back and fix it
|
|
172
|
+
// Implementation is optimized for loading/saving _whole files_.
|
|
173
|
+
/*
|
|
174
|
+
export class FileDBTransaction implements DBTransaction {
|
|
175
|
+
constructor(private db: FileDB) {}
|
|
176
|
+
|
|
177
|
+
ops: DBOperation[] = []
|
|
178
|
+
|
|
179
|
+
async commit(): Promise<void> {
|
|
180
|
+
// data[table][id] => row
|
|
181
|
+
const data: StringMap<StringMap<ObjectWithId>> = {}
|
|
182
|
+
|
|
183
|
+
// 1. Load all tables data (concurrently)
|
|
184
|
+
const tables = _uniq(this.ops.map(o => o.table))
|
|
185
|
+
|
|
186
|
+
await pMap(
|
|
187
|
+
tables,
|
|
188
|
+
async table => {
|
|
189
|
+
const rows = await this.db.loadFile(table)
|
|
190
|
+
data[table] = _by(rows, r => r.id)
|
|
191
|
+
},
|
|
192
|
+
{ concurrency: 16 },
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
const backup = _deepCopy(data)
|
|
196
|
+
|
|
197
|
+
// 2. Apply ops one by one (in order)
|
|
198
|
+
this.ops.forEach(op => {
|
|
199
|
+
if (op.type === 'deleteByIds') {
|
|
200
|
+
op.ids.forEach(id => delete data[op.table]![id])
|
|
201
|
+
} else if (op.type === 'saveBatch') {
|
|
202
|
+
op.rows.forEach(r => {
|
|
203
|
+
if (!r.id) {
|
|
204
|
+
throw new Error('FileDB: row has an empty id')
|
|
205
|
+
}
|
|
206
|
+
data[op.table]![r.id] = r
|
|
207
|
+
})
|
|
208
|
+
} else {
|
|
209
|
+
throw new Error(`DBOperation not supported: ${(op as any).type}`)
|
|
210
|
+
}
|
|
211
|
+
})
|
|
212
|
+
|
|
213
|
+
// 3. Sort, turn it into ops
|
|
214
|
+
// Not filtering empty arrays, cause it's already filtered in this.saveFiles()
|
|
215
|
+
const ops: DBSaveBatchOperation[] = _stringMapEntries(data).map(([table, map]) => {
|
|
216
|
+
return {
|
|
217
|
+
type: 'saveBatch',
|
|
218
|
+
table,
|
|
219
|
+
rows: this.db.sortRows(_stringMapValues(map)),
|
|
220
|
+
}
|
|
221
|
+
})
|
|
222
|
+
|
|
223
|
+
// 4. Save all files
|
|
224
|
+
try {
|
|
225
|
+
await this.db.saveFiles(ops)
|
|
226
|
+
} catch (err) {
|
|
227
|
+
const ops: DBSaveBatchOperation[] = _stringMapEntries(backup).map(([table, map]) => {
|
|
228
|
+
return {
|
|
229
|
+
type: 'saveBatch',
|
|
230
|
+
table,
|
|
231
|
+
rows: this.db.sortRows(_stringMapValues(map)),
|
|
232
|
+
}
|
|
233
|
+
})
|
|
234
|
+
|
|
235
|
+
// Rollback, ignore rollback error (if any)
|
|
236
|
+
await this.db.saveFiles(ops).catch(_ => {})
|
|
237
|
+
|
|
238
|
+
throw err
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
async rollback(): Promise<void> {
|
|
243
|
+
this.ops = []
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
*/
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { JsonSchemaObject, StringMap, JsonSchemaRootObject, ObjectWithId, CommonLogger } from '@naturalcycles/js-lib';
|
|
2
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
-
import { CommonDB, DBPatch,
|
|
4
|
-
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, RunQueryResult } from '../../db.model';
|
|
3
|
+
import { CommonDB, CommonDBType, DBOperation, DBPatch, DBTransactionFn } from '../..';
|
|
4
|
+
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, DBTransaction, RunQueryResult } from '../../db.model';
|
|
5
5
|
import { DBQuery } from '../../query/dbQuery';
|
|
6
6
|
export interface InMemoryDBCfg {
|
|
7
7
|
/**
|
|
@@ -36,6 +36,24 @@ export interface InMemoryDBCfg {
|
|
|
36
36
|
logger?: CommonLogger;
|
|
37
37
|
}
|
|
38
38
|
export declare class InMemoryDB implements CommonDB {
|
|
39
|
+
dbType: CommonDBType;
|
|
40
|
+
support: {
|
|
41
|
+
queries?: boolean | undefined;
|
|
42
|
+
dbQueryFilter?: boolean | undefined;
|
|
43
|
+
dbQueryFilterIn?: boolean | undefined;
|
|
44
|
+
dbQueryOrder?: boolean | undefined;
|
|
45
|
+
dbQuerySelectFields?: boolean | undefined;
|
|
46
|
+
insertSaveMethod?: boolean | undefined;
|
|
47
|
+
updateSaveMethod?: boolean | undefined;
|
|
48
|
+
updateByQuery?: boolean | undefined;
|
|
49
|
+
dbIncrement?: boolean | undefined;
|
|
50
|
+
createTable?: boolean | undefined;
|
|
51
|
+
tableSchemas?: boolean | undefined;
|
|
52
|
+
streaming?: boolean | undefined;
|
|
53
|
+
bufferValues?: boolean | undefined;
|
|
54
|
+
nullValues?: boolean | undefined;
|
|
55
|
+
transactions?: boolean | undefined;
|
|
56
|
+
};
|
|
39
57
|
constructor(cfg?: Partial<InMemoryDBCfg>);
|
|
40
58
|
cfg: InMemoryDBCfg;
|
|
41
59
|
data: StringMap<StringMap<ObjectWithId>>;
|
|
@@ -55,11 +73,12 @@ export declare class InMemoryDB implements CommonDB {
|
|
|
55
73
|
getByIds<ROW extends ObjectWithId>(_table: string, ids: ROW['id'][], _opt?: CommonDBOptions): Promise<ROW[]>;
|
|
56
74
|
saveBatch<ROW extends Partial<ObjectWithId>>(_table: string, rows: ROW[], opt?: CommonDBSaveOptions<ROW>): Promise<void>;
|
|
57
75
|
deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
|
|
76
|
+
deleteByIds(_table: string, ids: string[], _opt?: CommonDBOptions): Promise<number>;
|
|
58
77
|
updateByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, patch: DBPatch<ROW>): Promise<number>;
|
|
59
78
|
runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<RunQueryResult<ROW>>;
|
|
60
79
|
runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
|
|
61
80
|
streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): ReadableTyped<ROW>;
|
|
62
|
-
|
|
81
|
+
runInTransaction(fn: DBTransactionFn): Promise<void>;
|
|
63
82
|
/**
|
|
64
83
|
* Flushes all tables (all namespaces) at once.
|
|
65
84
|
*/
|
|
@@ -69,3 +88,13 @@ export declare class InMemoryDB implements CommonDB {
|
|
|
69
88
|
*/
|
|
70
89
|
restoreFromDisk(): Promise<void>;
|
|
71
90
|
}
|
|
91
|
+
export declare class InMemoryDBTransaction implements DBTransaction {
|
|
92
|
+
private db;
|
|
93
|
+
constructor(db: InMemoryDB);
|
|
94
|
+
ops: DBOperation[];
|
|
95
|
+
getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: CommonDBOptions): Promise<ROW[]>;
|
|
96
|
+
saveBatch<ROW extends Partial<ObjectWithId>>(table: string, rows: ROW[], opt?: CommonDBSaveOptions<ROW>): Promise<void>;
|
|
97
|
+
deleteByIds(table: string, ids: string[], opt?: CommonDBOptions): Promise<number>;
|
|
98
|
+
commit(): Promise<void>;
|
|
99
|
+
rollback(): Promise<void>;
|
|
100
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.InMemoryDB = void 0;
|
|
3
|
+
exports.InMemoryDBTransaction = exports.InMemoryDB = void 0;
|
|
4
4
|
const tslib_1 = require("tslib");
|
|
5
5
|
const node_fs_1 = tslib_1.__importDefault(require("node:fs"));
|
|
6
6
|
const promises_1 = tslib_1.__importDefault(require("node:fs/promises"));
|
|
@@ -9,9 +9,12 @@ const node_zlib_1 = require("node:zlib");
|
|
|
9
9
|
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
10
10
|
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
11
11
|
const __1 = require("../..");
|
|
12
|
-
const dbQuery_1 = require("../../query/dbQuery");
|
|
13
12
|
class InMemoryDB {
|
|
14
13
|
constructor(cfg) {
|
|
14
|
+
this.dbType = __1.CommonDBType.document;
|
|
15
|
+
this.support = {
|
|
16
|
+
...__1.commonDBFullSupport,
|
|
17
|
+
};
|
|
15
18
|
// data[table][id] > {id: 'a', created: ... }
|
|
16
19
|
this.data = {};
|
|
17
20
|
this.cfg = {
|
|
@@ -95,12 +98,20 @@ class InMemoryDB {
|
|
|
95
98
|
}
|
|
96
99
|
async deleteByQuery(q, _opt) {
|
|
97
100
|
const table = this.cfg.tablesPrefix + q.table;
|
|
98
|
-
this.data[table]
|
|
101
|
+
if (!this.data[table])
|
|
102
|
+
return 0;
|
|
103
|
+
const ids = (0, __1.queryInMemory)(q, Object.values(this.data[table])).map(r => r.id);
|
|
104
|
+
return await this.deleteByIds(q.table, ids);
|
|
105
|
+
}
|
|
106
|
+
async deleteByIds(_table, ids, _opt) {
|
|
107
|
+
const table = this.cfg.tablesPrefix + _table;
|
|
108
|
+
if (!this.data[table])
|
|
109
|
+
return 0;
|
|
99
110
|
let count = 0;
|
|
100
|
-
|
|
101
|
-
if (!this.data[table][
|
|
111
|
+
ids.forEach(id => {
|
|
112
|
+
if (!this.data[table][id])
|
|
102
113
|
return;
|
|
103
|
-
delete this.data[table][
|
|
114
|
+
delete this.data[table][id];
|
|
104
115
|
count++;
|
|
105
116
|
});
|
|
106
117
|
return count;
|
|
@@ -135,28 +146,14 @@ class InMemoryDB {
|
|
|
135
146
|
const table = this.cfg.tablesPrefix + q.table;
|
|
136
147
|
return node_stream_1.Readable.from((0, __1.queryInMemory)(q, Object.values(this.data[table] || {})));
|
|
137
148
|
}
|
|
138
|
-
async
|
|
139
|
-
const
|
|
149
|
+
async runInTransaction(fn) {
|
|
150
|
+
const tx = new InMemoryDBTransaction(this);
|
|
140
151
|
try {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt });
|
|
144
|
-
}
|
|
145
|
-
else if (op.type === 'deleteByIds') {
|
|
146
|
-
await this.deleteByQuery(dbQuery_1.DBQuery.create(op.table).filter('id', 'in', op.ids), {
|
|
147
|
-
...op.opt,
|
|
148
|
-
...opt,
|
|
149
|
-
});
|
|
150
|
-
}
|
|
151
|
-
else {
|
|
152
|
-
throw new Error(`DBOperation not supported: ${op.type}`);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
152
|
+
await fn(tx);
|
|
153
|
+
await tx.commit();
|
|
155
154
|
}
|
|
156
155
|
catch (err) {
|
|
157
|
-
|
|
158
|
-
this.data = backup;
|
|
159
|
-
this.cfg.logger.log('InMemoryDB transaction rolled back');
|
|
156
|
+
await tx.rollback();
|
|
160
157
|
throw err;
|
|
161
158
|
}
|
|
162
159
|
}
|
|
@@ -215,3 +212,57 @@ class InMemoryDB {
|
|
|
215
212
|
}
|
|
216
213
|
}
|
|
217
214
|
exports.InMemoryDB = InMemoryDB;
|
|
215
|
+
class InMemoryDBTransaction {
|
|
216
|
+
constructor(db) {
|
|
217
|
+
this.db = db;
|
|
218
|
+
this.ops = [];
|
|
219
|
+
}
|
|
220
|
+
async getByIds(table, ids, opt) {
|
|
221
|
+
return await this.db.getByIds(table, ids, opt);
|
|
222
|
+
}
|
|
223
|
+
async saveBatch(table, rows, opt) {
|
|
224
|
+
this.ops.push({
|
|
225
|
+
type: 'saveBatch',
|
|
226
|
+
table,
|
|
227
|
+
rows,
|
|
228
|
+
opt,
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
async deleteByIds(table, ids, opt) {
|
|
232
|
+
this.ops.push({
|
|
233
|
+
type: 'deleteByIds',
|
|
234
|
+
table,
|
|
235
|
+
ids,
|
|
236
|
+
opt,
|
|
237
|
+
});
|
|
238
|
+
return ids.length;
|
|
239
|
+
}
|
|
240
|
+
async commit() {
|
|
241
|
+
const backup = (0, js_lib_1._deepCopy)(this.db.data);
|
|
242
|
+
try {
|
|
243
|
+
for (const op of this.ops) {
|
|
244
|
+
if (op.type === 'saveBatch') {
|
|
245
|
+
await this.db.saveBatch(op.table, op.rows, op.opt);
|
|
246
|
+
}
|
|
247
|
+
else if (op.type === 'deleteByIds') {
|
|
248
|
+
await this.db.deleteByIds(op.table, op.ids, op.opt);
|
|
249
|
+
}
|
|
250
|
+
else {
|
|
251
|
+
throw new Error(`DBOperation not supported: ${op.type}`);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
this.ops = [];
|
|
255
|
+
}
|
|
256
|
+
catch (err) {
|
|
257
|
+
// rollback
|
|
258
|
+
this.ops = [];
|
|
259
|
+
this.db.data = backup;
|
|
260
|
+
this.db.cfg.logger.log('InMemoryDB transaction rolled back');
|
|
261
|
+
throw err;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
async rollback() {
|
|
265
|
+
this.ops = [];
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
exports.InMemoryDBTransaction = InMemoryDBTransaction;
|
package/dist/base.common.db.d.ts
CHANGED
|
@@ -1,29 +1,26 @@
|
|
|
1
1
|
import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
-
import { CommonDB } from './common.db';
|
|
4
|
-
import { CommonDBOptions, CommonDBSaveOptions, DBPatch, RunQueryResult } from './db.model';
|
|
3
|
+
import { CommonDB, CommonDBSupport, CommonDBType } from './common.db';
|
|
4
|
+
import { CommonDBOptions, CommonDBSaveOptions, DBPatch, DBTransactionFn, RunQueryResult } from './db.model';
|
|
5
5
|
import { DBQuery } from './query/dbQuery';
|
|
6
|
-
import { DBTransaction } from './transaction/dbTransaction';
|
|
7
6
|
/**
|
|
8
7
|
* No-op implementation of CommonDB interface.
|
|
9
8
|
* To be extended by actual implementations.
|
|
10
9
|
*/
|
|
11
10
|
export declare class BaseCommonDB implements CommonDB {
|
|
11
|
+
dbType: CommonDBType;
|
|
12
|
+
support: CommonDBSupport;
|
|
12
13
|
ping(): Promise<void>;
|
|
13
14
|
getTables(): Promise<string[]>;
|
|
14
15
|
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;
|
|
15
16
|
createTable<ROW extends ObjectWithId>(table: string, schema: JsonSchemaObject<ROW>): Promise<void>;
|
|
16
|
-
getByIds<ROW extends ObjectWithId>(table: string, ids:
|
|
17
|
+
getByIds<ROW extends ObjectWithId>(table: string, ids: string[]): Promise<ROW[]>;
|
|
17
18
|
deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>): Promise<number>;
|
|
18
19
|
updateByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, patch: DBPatch<ROW>, opt?: CommonDBOptions): Promise<number>;
|
|
19
20
|
runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>): Promise<RunQueryResult<ROW>>;
|
|
20
21
|
runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>): Promise<number>;
|
|
21
22
|
saveBatch<ROW extends Partial<ObjectWithId>>(table: string, rows: ROW[], opt?: CommonDBSaveOptions<ROW>): Promise<void>;
|
|
22
23
|
streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>): ReadableTyped<ROW>;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
* Doesn't support rollback on error, hence doesn't pass dbTest.
|
|
26
|
-
* To be extended.
|
|
27
|
-
*/
|
|
28
|
-
commitTransaction(tx: DBTransaction, opt?: CommonDBOptions): Promise<void>;
|
|
24
|
+
deleteByIds(table: string, ids: string[], opt?: CommonDBOptions): Promise<number>;
|
|
25
|
+
runInTransaction(fn: DBTransactionFn): Promise<void>;
|
|
29
26
|
}
|
package/dist/base.common.db.js
CHANGED
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BaseCommonDB = void 0;
|
|
4
|
+
const common_db_1 = require("./common.db");
|
|
5
|
+
const dbTransaction_util_1 = require("./transaction/dbTransaction.util");
|
|
4
6
|
/* eslint-disable unused-imports/no-unused-vars */
|
|
5
7
|
/**
|
|
6
8
|
* No-op implementation of CommonDB interface.
|
|
7
9
|
* To be extended by actual implementations.
|
|
8
10
|
*/
|
|
9
11
|
class BaseCommonDB {
|
|
12
|
+
constructor() {
|
|
13
|
+
this.dbType = common_db_1.CommonDBType.document;
|
|
14
|
+
this.support = {};
|
|
15
|
+
}
|
|
10
16
|
async ping() {
|
|
11
17
|
throw new Error('ping is not implemented');
|
|
12
18
|
}
|
|
@@ -40,13 +46,13 @@ class BaseCommonDB {
|
|
|
40
46
|
streamQuery(q) {
|
|
41
47
|
throw new Error('streamQuery is not implemented');
|
|
42
48
|
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
async deleteByIds(table, ids, opt) {
|
|
50
|
+
throw new Error('deleteByIds is not implemented');
|
|
51
|
+
}
|
|
52
|
+
async runInTransaction(fn) {
|
|
53
|
+
const tx = new dbTransaction_util_1.FakeDBTransaction(this);
|
|
54
|
+
await fn(tx);
|
|
55
|
+
// there's no try/catch and rollback, as there's nothing to rollback
|
|
50
56
|
}
|
|
51
57
|
}
|
|
52
58
|
exports.BaseCommonDB = BaseCommonDB;
|
package/dist/common.db.d.ts
CHANGED
|
@@ -1,9 +1,35 @@
|
|
|
1
1
|
import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
-
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
-
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, DBPatch, RunQueryResult } from './db.model';
|
|
2
|
+
import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
+
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, DBPatch, DBTransactionFn, RunQueryResult } from './db.model';
|
|
4
4
|
import { DBQuery } from './query/dbQuery';
|
|
5
|
-
|
|
5
|
+
export declare enum CommonDBType {
|
|
6
|
+
'document' = "document",
|
|
7
|
+
'relational' = "relational"
|
|
8
|
+
}
|
|
6
9
|
export interface CommonDB {
|
|
10
|
+
/**
|
|
11
|
+
* Relational databases are expected to return `null` for all missing properties.
|
|
12
|
+
*/
|
|
13
|
+
dbType: CommonDBType;
|
|
14
|
+
/**
|
|
15
|
+
* Manifest of supported features.
|
|
16
|
+
*/
|
|
17
|
+
support: CommonDBSupport;
|
|
18
|
+
supportsQueries?: boolean;
|
|
19
|
+
supportsDBQueryFilter?: boolean;
|
|
20
|
+
supportsDBQueryFilterIn?: boolean;
|
|
21
|
+
supportsDBQueryOrder?: boolean;
|
|
22
|
+
supportsDBQuerySelectFields?: boolean;
|
|
23
|
+
supportsInsertSaveMethod?: boolean;
|
|
24
|
+
supportsUpdateSaveMethod?: boolean;
|
|
25
|
+
supportsUpdateByQuery?: boolean;
|
|
26
|
+
supportsDBIncrement?: boolean;
|
|
27
|
+
supportsCreateTable?: boolean;
|
|
28
|
+
supportsTableSchemas?: boolean;
|
|
29
|
+
supportsStreaming?: boolean;
|
|
30
|
+
supportsBufferValues?: boolean;
|
|
31
|
+
supportsNullValues?: boolean;
|
|
32
|
+
supportsTransactions?: boolean;
|
|
7
33
|
/**
|
|
8
34
|
* Checks that connection/credentials/etc is ok.
|
|
9
35
|
* Also acts as a "warmup request" for a DB.
|
|
@@ -42,6 +68,11 @@ export interface CommonDB {
|
|
|
42
68
|
* rows can have missing ids only if DB supports auto-generating them (like mysql auto_increment).
|
|
43
69
|
*/
|
|
44
70
|
saveBatch: <ROW extends Partial<ObjectWithId>>(table: string, rows: ROW[], opt?: CommonDBSaveOptions<ROW>) => Promise<void>;
|
|
71
|
+
/**
|
|
72
|
+
* Returns number of deleted items.
|
|
73
|
+
* Not supported by all implementations (e.g Datastore will always return same number as number of ids).
|
|
74
|
+
*/
|
|
75
|
+
deleteByIds: (table: string, ids: string[], opt?: CommonDBOptions) => Promise<number>;
|
|
45
76
|
/**
|
|
46
77
|
* Returns number of deleted items.
|
|
47
78
|
* Not supported by all implementations (e.g Datastore will always return same number as number of ids).
|
|
@@ -69,6 +100,31 @@ export interface CommonDB {
|
|
|
69
100
|
/**
|
|
70
101
|
* Should be implemented as a Transaction (best effort), which means that
|
|
71
102
|
* either ALL or NONE of the operations should be applied.
|
|
103
|
+
*
|
|
104
|
+
* Transaction is automatically committed if fn resolves normally.
|
|
105
|
+
* Transaction is rolled back if fn throws, the error is re-thrown in that case.
|
|
106
|
+
* Graceful rollback is allowed on tx.rollback()
|
|
72
107
|
*/
|
|
73
|
-
|
|
108
|
+
runInTransaction: (fn: DBTransactionFn) => Promise<void>;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Manifest of supported features.
|
|
112
|
+
*/
|
|
113
|
+
export interface CommonDBSupport {
|
|
114
|
+
queries?: boolean;
|
|
115
|
+
dbQueryFilter?: boolean;
|
|
116
|
+
dbQueryFilterIn?: boolean;
|
|
117
|
+
dbQueryOrder?: boolean;
|
|
118
|
+
dbQuerySelectFields?: boolean;
|
|
119
|
+
insertSaveMethod?: boolean;
|
|
120
|
+
updateSaveMethod?: boolean;
|
|
121
|
+
updateByQuery?: boolean;
|
|
122
|
+
dbIncrement?: boolean;
|
|
123
|
+
createTable?: boolean;
|
|
124
|
+
tableSchemas?: boolean;
|
|
125
|
+
streaming?: boolean;
|
|
126
|
+
bufferValues?: boolean;
|
|
127
|
+
nullValues?: boolean;
|
|
128
|
+
transactions?: boolean;
|
|
74
129
|
}
|
|
130
|
+
export declare const commonDBFullSupport: CommonDBSupport;
|