@naturalcycles/db-lib 8.11.0 → 8.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/adapter/cachedb/cache.db.d.ts +3 -3
- package/dist/adapter/cachedb/cache.db.js +4 -4
- package/dist/adapter/file/file.db.d.ts +2 -2
- package/dist/adapter/file/file.db.js +20 -18
- package/dist/adapter/file/inMemory.persistence.plugin.js +1 -1
- package/dist/adapter/file/localFile.persistence.plugin.js +9 -9
- package/dist/adapter/inmemory/inMemory.db.d.ts +3 -4
- package/dist/adapter/inmemory/inMemory.db.js +25 -23
- package/dist/adapter/inmemory/queryInMemory.js +1 -1
- package/dist/base.common.db.d.ts +3 -3
- package/dist/base.common.db.js +10 -4
- package/dist/common.db.d.ts +3 -3
- package/dist/commondao/common.dao.d.ts +7 -7
- package/dist/commondao/common.dao.js +56 -48
- package/dist/commondao/common.dao.model.d.ts +20 -27
- package/dist/db.model.js +2 -2
- package/dist/getDB.js +3 -3
- package/dist/index.d.ts +2 -4
- package/dist/index.js +1 -5
- package/dist/kv/commonKeyValueDao.js +4 -4
- package/dist/model.util.js +2 -2
- package/dist/pipeline/dbPipelineBackup.d.ts +0 -1
- package/dist/pipeline/dbPipelineBackup.js +14 -25
- package/dist/pipeline/dbPipelineCopy.js +11 -11
- package/dist/pipeline/dbPipelineRestore.js +20 -20
- package/dist/query/dbQuery.js +2 -2
- package/dist/testing/daoTest.js +23 -23
- package/dist/testing/dbTest.js +17 -17
- package/dist/testing/keyValueDBTest.js +18 -14
- package/dist/testing/keyValueDaoTest.js +18 -14
- package/dist/testing/test.model.d.ts +4 -2
- package/dist/testing/test.model.js +39 -8
- package/dist/testing/timeSeriesTest.util.js +1 -1
- package/dist/transaction/dbTransaction.util.js +2 -2
- package/dist/validation/index.js +9 -9
- package/package.json +1 -1
- package/readme.md +5 -4
- package/src/adapter/cachedb/cache.db.ts +9 -5
- package/src/adapter/file/file.db.ts +7 -4
- package/src/adapter/inmemory/inMemory.db.ts +20 -7
- package/src/base.common.db.ts +10 -4
- package/src/common.db.ts +3 -3
- package/src/commondao/common.dao.model.ts +24 -29
- package/src/commondao/common.dao.ts +35 -24
- package/src/index.ts +0 -7
- package/src/pipeline/dbPipelineBackup.ts +2 -15
- package/src/pipeline/dbPipelineRestore.ts +1 -1
- package/src/testing/dbTest.ts +1 -1
- package/src/testing/keyValueDBTest.ts +10 -6
- package/src/testing/keyValueDaoTest.ts +10 -6
- package/src/testing/test.model.ts +38 -4
- package/dist/schema/common.schema.d.ts +0 -38
- package/dist/schema/common.schema.js +0 -18
- package/dist/schema/commonSchemaGenerator.d.ts +0 -35
- package/dist/schema/commonSchemaGenerator.js +0 -151
- package/src/schema/common.schema.ts +0 -49
- package/src/schema/commonSchemaGenerator.ts +0 -200
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,32 @@
|
|
|
1
|
+
# [8.13.0](https://github.com/NaturalCycles/db-lib/compare/v8.12.0...v8.13.0) (2021-10-01)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* remove CommonSchema in favor of JsonSchema ([1827ec5](https://github.com/NaturalCycles/db-lib/commit/1827ec5709053b97a6b9e131f2ec0b40176c3e23))
|
|
7
|
+
|
|
8
|
+
# [8.12.0](https://github.com/NaturalCycles/db-lib/compare/v8.11.2...v8.12.0) (2021-09-10)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Features
|
|
12
|
+
|
|
13
|
+
* CommonDao hooks return types are loosened to Partial ([141bd29](https://github.com/NaturalCycles/db-lib/commit/141bd29166676eee1bc9bd72f23cd5b88dc22f3b))
|
|
14
|
+
* remove `throwOnError*` CommonDao options ([008674f](https://github.com/NaturalCycles/db-lib/commit/008674fc7a755cbb2e540250d9e752c7db2faeb4))
|
|
15
|
+
|
|
16
|
+
## [8.11.2](https://github.com/NaturalCycles/db-lib/compare/v8.11.1...v8.11.2) (2021-08-24)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
### Bug Fixes
|
|
20
|
+
|
|
21
|
+
* CommonDaoStreamForEachOptions to extend TransformMapOpt ([dd8e5b5](https://github.com/NaturalCycles/db-lib/commit/dd8e5b5e0e5d20897182ecda7c2c70e723b5e2bb))
|
|
22
|
+
|
|
23
|
+
## [8.11.1](https://github.com/NaturalCycles/db-lib/compare/v8.11.0...v8.11.1) (2021-08-24)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
### Bug Fixes
|
|
27
|
+
|
|
28
|
+
* kvDBTest to not rely on deterministic output ([d150e44](https://github.com/NaturalCycles/db-lib/commit/d150e4499a7f2a5f9c266b153628826e5ebb6e96))
|
|
29
|
+
|
|
1
30
|
# [8.11.0](https://github.com/NaturalCycles/db-lib/compare/v8.10.0...v8.11.0) (2021-08-24)
|
|
2
31
|
|
|
3
32
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/// <reference types="node" />
|
|
2
|
+
import { JsonSchemaObject } from '@naturalcycles/js-lib';
|
|
2
3
|
import { IDebugger } from '@naturalcycles/nodejs-lib';
|
|
3
4
|
import { Readable } from 'stream';
|
|
4
5
|
import { BaseCommonDB } from '../../base.common.db';
|
|
5
6
|
import { CommonDB } from '../../common.db';
|
|
6
7
|
import { ObjectWithId, RunQueryResult } from '../../db.model';
|
|
7
8
|
import { DBQuery } from '../../query/dbQuery';
|
|
8
|
-
import { CommonSchema } from '../../schema/common.schema';
|
|
9
9
|
import { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBStreamOptions } from './cache.db.model';
|
|
10
10
|
/**
|
|
11
11
|
* CommonDB implementation that proxies requests to downstream CommonDB
|
|
@@ -22,8 +22,8 @@ export declare class CacheDB extends BaseCommonDB implements CommonDB {
|
|
|
22
22
|
* Resets InMemory DB data
|
|
23
23
|
*/
|
|
24
24
|
getTables(): Promise<string[]>;
|
|
25
|
-
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<
|
|
26
|
-
createTable(schema:
|
|
25
|
+
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaObject<ROW>>;
|
|
26
|
+
createTable(table: string, schema: JsonSchemaObject, opt?: CacheDBCreateOptions): Promise<void>;
|
|
27
27
|
getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: CacheDBOptions): Promise<ROW[]>;
|
|
28
28
|
deleteByIds(table: string, ids: string[], opt?: CacheDBOptions): Promise<number>;
|
|
29
29
|
saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: CacheDBOptions): Promise<void>;
|
|
@@ -14,7 +14,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
|
|
|
14
14
|
constructor(cfg) {
|
|
15
15
|
super();
|
|
16
16
|
this.cfg = cfg;
|
|
17
|
-
this.log = nodejs_lib_1.Debug(`nc:db-lib:${cfg.name}`);
|
|
17
|
+
this.log = (0, nodejs_lib_1.Debug)(`nc:db-lib:${cfg.name}`);
|
|
18
18
|
}
|
|
19
19
|
async ping() {
|
|
20
20
|
await Promise.all([this.cfg.cacheDB.ping(), this.cfg.downstreamDB.ping()]);
|
|
@@ -33,12 +33,12 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
|
|
|
33
33
|
async getTableSchema(table) {
|
|
34
34
|
return await this.cfg.downstreamDB.getTableSchema(table);
|
|
35
35
|
}
|
|
36
|
-
async createTable(schema, opt = {}) {
|
|
36
|
+
async createTable(table, schema, opt = {}) {
|
|
37
37
|
if (!opt.onlyCache && !this.cfg.onlyCache) {
|
|
38
|
-
await this.cfg.downstreamDB.createTable(schema, opt);
|
|
38
|
+
await this.cfg.downstreamDB.createTable(table, schema, opt);
|
|
39
39
|
}
|
|
40
40
|
if (!opt.skipCache && !this.cfg.skipCache) {
|
|
41
|
-
await this.cfg.cacheDB.createTable(schema, opt);
|
|
41
|
+
await this.cfg.cacheDB.createTable(table, schema, opt);
|
|
42
42
|
}
|
|
43
43
|
}
|
|
44
44
|
async getByIds(table, ids, opt = {}) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
import { JsonSchemaObject } from '@naturalcycles/js-lib';
|
|
1
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
2
3
|
import { BaseCommonDB, DBSaveBatchOperation, ObjectWithId } from '../..';
|
|
3
|
-
import { CommonSchema } from '../..';
|
|
4
4
|
import { CommonDB } from '../../common.db';
|
|
5
5
|
import { CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, RunQueryResult } from '../../db.model';
|
|
6
6
|
import { DBQuery } from '../../query/dbQuery';
|
|
@@ -32,7 +32,7 @@ export declare class FileDB extends BaseCommonDB implements CommonDB {
|
|
|
32
32
|
streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: CommonDBStreamOptions): ReadableTyped<ROW>;
|
|
33
33
|
deleteByIds<ROW extends ObjectWithId>(table: string, ids: string[], _opt?: CommonDBOptions): Promise<number>;
|
|
34
34
|
deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
|
|
35
|
-
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<
|
|
35
|
+
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaObject<ROW>>;
|
|
36
36
|
loadFile<ROW extends ObjectWithId>(table: string): Promise<ROW[]>;
|
|
37
37
|
saveFile<ROW extends ObjectWithId>(table: string, _rows: ROW[]): Promise<void>;
|
|
38
38
|
saveFiles(ops: DBSaveBatchOperation[]): Promise<void>;
|
|
@@ -5,8 +5,7 @@ const js_lib_1 = require("@naturalcycles/js-lib");
|
|
|
5
5
|
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
6
6
|
const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
|
|
7
7
|
const __1 = require("../..");
|
|
8
|
-
const
|
|
9
|
-
const log = nodejs_lib_1.Debug('nc:db-lib:filedb');
|
|
8
|
+
const log = (0, nodejs_lib_1.Debug)('nc:db-lib:filedb');
|
|
10
9
|
/**
|
|
11
10
|
* Provides barebone implementation for "whole file" based CommonDB.
|
|
12
11
|
* "whole file" means that the persistence layer doesn't allow any querying,
|
|
@@ -36,18 +35,18 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
36
35
|
return tables;
|
|
37
36
|
}
|
|
38
37
|
async getByIds(table, ids, _opt) {
|
|
39
|
-
const byId = js_lib_1._by(await this.loadFile(table), r => r.id);
|
|
38
|
+
const byId = (0, js_lib_1._by)(await this.loadFile(table), r => r.id);
|
|
40
39
|
return ids.map(id => byId[id]).filter(Boolean);
|
|
41
40
|
}
|
|
42
41
|
async saveBatch(table, rows, _opt) {
|
|
43
42
|
if (!rows.length)
|
|
44
43
|
return; // save some api calls
|
|
45
44
|
// 1. Load the whole file from gh
|
|
46
|
-
const byId = js_lib_1._by(await this.loadFile(table), r => r.id);
|
|
45
|
+
const byId = (0, js_lib_1._by)(await this.loadFile(table), r => r.id);
|
|
47
46
|
// 2. Merge with new data (using ids)
|
|
48
47
|
let saved = 0;
|
|
49
48
|
rows.forEach(r => {
|
|
50
|
-
if (!js_lib_1._deepEquals(byId[r.id], r)) {
|
|
49
|
+
if (!(0, js_lib_1._deepEquals)(byId[r.id], r)) {
|
|
51
50
|
byId[r.id] = r;
|
|
52
51
|
saved++;
|
|
53
52
|
}
|
|
@@ -55,7 +54,7 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
55
54
|
// Only save if there are changed rows
|
|
56
55
|
if (saved > 0) {
|
|
57
56
|
// 3. Save the whole file
|
|
58
|
-
await this.saveFile(table, js_lib_1._stringMapValues(byId));
|
|
57
|
+
await this.saveFile(table, (0, js_lib_1._stringMapValues)(byId));
|
|
59
58
|
}
|
|
60
59
|
}
|
|
61
60
|
/**
|
|
@@ -65,10 +64,10 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
65
64
|
// data[table][id] => row
|
|
66
65
|
const data = {};
|
|
67
66
|
// 1. Load all tables data (concurrently)
|
|
68
|
-
const tables = js_lib_1._uniq(tx.ops.map(o => o.table));
|
|
69
|
-
await js_lib_1.pMap(tables, async (table) => {
|
|
67
|
+
const tables = (0, js_lib_1._uniq)(tx.ops.map(o => o.table));
|
|
68
|
+
await (0, js_lib_1.pMap)(tables, async (table) => {
|
|
70
69
|
const rows = await this.loadFile(table);
|
|
71
|
-
data[table] = js_lib_1._by(rows, r => r.id);
|
|
70
|
+
data[table] = (0, js_lib_1._by)(rows, r => r.id);
|
|
72
71
|
}, { concurrency: 16 });
|
|
73
72
|
// 2. Apply ops one by one (in order)
|
|
74
73
|
tx.ops.forEach(op => {
|
|
@@ -96,14 +95,14 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
96
95
|
}
|
|
97
96
|
async runQuery(q, _opt) {
|
|
98
97
|
return {
|
|
99
|
-
rows: __1.queryInMemory(q, await this.loadFile(q.table)),
|
|
98
|
+
rows: (0, __1.queryInMemory)(q, await this.loadFile(q.table)),
|
|
100
99
|
};
|
|
101
100
|
}
|
|
102
101
|
async runQueryCount(q, _opt) {
|
|
103
102
|
return (await this.loadFile(q.table)).length;
|
|
104
103
|
}
|
|
105
104
|
streamQuery(q, opt) {
|
|
106
|
-
const readable = nodejs_lib_1.readableCreate();
|
|
105
|
+
const readable = (0, nodejs_lib_1.readableCreate)();
|
|
107
106
|
void this.runQuery(q, opt).then(({ rows }) => {
|
|
108
107
|
rows.forEach(r => readable.push(r));
|
|
109
108
|
readable.push(null); // done
|
|
@@ -127,20 +126,23 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
127
126
|
return deleted;
|
|
128
127
|
}
|
|
129
128
|
async deleteByQuery(q, _opt) {
|
|
130
|
-
const byId = js_lib_1._by(await this.loadFile(q.table), r => r.id);
|
|
129
|
+
const byId = (0, js_lib_1._by)(await this.loadFile(q.table), r => r.id);
|
|
131
130
|
let deleted = 0;
|
|
132
|
-
__1.queryInMemory(q, js_lib_1._stringMapValues(byId)).forEach(r => {
|
|
131
|
+
(0, __1.queryInMemory)(q, (0, js_lib_1._stringMapValues)(byId)).forEach(r => {
|
|
133
132
|
delete byId[r.id];
|
|
134
133
|
deleted++;
|
|
135
134
|
});
|
|
136
135
|
if (deleted > 0) {
|
|
137
|
-
await this.saveFile(q.table, js_lib_1._stringMapValues(byId));
|
|
136
|
+
await this.saveFile(q.table, (0, js_lib_1._stringMapValues)(byId));
|
|
138
137
|
}
|
|
139
138
|
return deleted;
|
|
140
139
|
}
|
|
141
140
|
async getTableSchema(table) {
|
|
142
141
|
const rows = await this.loadFile(table);
|
|
143
|
-
return
|
|
142
|
+
return {
|
|
143
|
+
...(0, js_lib_1.generateJsonSchemaFromData)(rows),
|
|
144
|
+
$id: `${table}.schema.json`,
|
|
145
|
+
};
|
|
144
146
|
}
|
|
145
147
|
// wrapper, to handle logging
|
|
146
148
|
async loadFile(table) {
|
|
@@ -172,12 +174,12 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
172
174
|
*/
|
|
173
175
|
sortRows(rows) {
|
|
174
176
|
if (this.cfg.sortOnSave) {
|
|
175
|
-
js_lib_1._sortBy(rows, r => r[this.cfg.sortOnSave.name], true);
|
|
177
|
+
(0, js_lib_1._sortBy)(rows, r => r[this.cfg.sortOnSave.name], true);
|
|
176
178
|
if (this.cfg.sortOnSave.descending)
|
|
177
179
|
rows.reverse(); // mutates
|
|
178
180
|
}
|
|
179
181
|
if (this.cfg.sortObjects) {
|
|
180
|
-
return js_lib_1._sortObjectDeep(rows);
|
|
182
|
+
return (0, js_lib_1._sortObjectDeep)(rows);
|
|
181
183
|
}
|
|
182
184
|
return rows;
|
|
183
185
|
}
|
|
@@ -190,7 +192,7 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
190
192
|
logFinished(started, op) {
|
|
191
193
|
if (!this.cfg.logFinished)
|
|
192
194
|
return;
|
|
193
|
-
log(`<< ${op} ${colors_1.dimGrey(`in ${js_lib_1._since(started)}`)}`);
|
|
195
|
+
log(`<< ${op} ${(0, colors_1.dimGrey)(`in ${(0, js_lib_1._since)(started)}`)}`);
|
|
194
196
|
}
|
|
195
197
|
}
|
|
196
198
|
exports.FileDB = FileDB;
|
|
@@ -29,28 +29,28 @@ class LocalFilePersistencePlugin {
|
|
|
29
29
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
|
|
30
30
|
if (!(await fs.pathExists(filePath)))
|
|
31
31
|
return [];
|
|
32
|
-
const transformUnzip = this.cfg.gzip ? [zlib_1.createUnzip()] : [];
|
|
32
|
+
const transformUnzip = this.cfg.gzip ? [(0, zlib_1.createUnzip)()] : [];
|
|
33
33
|
const rows = [];
|
|
34
|
-
await nodejs_lib_1._pipeline([
|
|
34
|
+
await (0, nodejs_lib_1._pipeline)([
|
|
35
35
|
fs.createReadStream(filePath),
|
|
36
36
|
...transformUnzip,
|
|
37
|
-
nodejs_lib_1.transformSplit(),
|
|
38
|
-
nodejs_lib_1.transformJsonParse(),
|
|
39
|
-
nodejs_lib_1.writablePushToArray(rows),
|
|
37
|
+
(0, nodejs_lib_1.transformSplit)(),
|
|
38
|
+
(0, nodejs_lib_1.transformJsonParse)(),
|
|
39
|
+
(0, nodejs_lib_1.writablePushToArray)(rows),
|
|
40
40
|
]);
|
|
41
41
|
return rows;
|
|
42
42
|
}
|
|
43
43
|
async saveFiles(ops) {
|
|
44
|
-
await js_lib_1.pMap(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 16 });
|
|
44
|
+
await (0, js_lib_1.pMap)(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 16 });
|
|
45
45
|
}
|
|
46
46
|
async saveFile(table, rows) {
|
|
47
47
|
await fs.ensureDir(this.cfg.storagePath);
|
|
48
48
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
|
|
49
49
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
|
|
50
|
-
const transformZip = this.cfg.gzip ? [zlib_1.createGzip()] : [];
|
|
51
|
-
await nodejs_lib_1._pipeline([
|
|
50
|
+
const transformZip = this.cfg.gzip ? [(0, zlib_1.createGzip)()] : [];
|
|
51
|
+
await (0, nodejs_lib_1._pipeline)([
|
|
52
52
|
stream_1.Readable.from(rows),
|
|
53
|
-
nodejs_lib_1.transformToNDJson(),
|
|
53
|
+
(0, nodejs_lib_1.transformToNDJson)(),
|
|
54
54
|
...transformZip,
|
|
55
55
|
fs.createWriteStream(filePath),
|
|
56
56
|
]);
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { StringMap } from '@naturalcycles/js-lib';
|
|
1
|
+
import { JsonSchemaObject, StringMap } from '@naturalcycles/js-lib';
|
|
2
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
3
|
import { CommonDB, DBTransaction, ObjectWithId } from '../..';
|
|
4
|
-
import { CommonSchema } from '../..';
|
|
5
4
|
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, RunQueryResult } from '../../db.model';
|
|
6
5
|
import { DBQuery } from '../../query/dbQuery';
|
|
7
6
|
export interface InMemoryDBCfg {
|
|
@@ -47,8 +46,8 @@ export declare class InMemoryDB implements CommonDB {
|
|
|
47
46
|
*/
|
|
48
47
|
resetCache(_table?: string): Promise<void>;
|
|
49
48
|
getTables(): Promise<string[]>;
|
|
50
|
-
getTableSchema<ROW extends ObjectWithId>(_table: string): Promise<
|
|
51
|
-
createTable(
|
|
49
|
+
getTableSchema<ROW extends ObjectWithId>(_table: string): Promise<JsonSchemaObject<ROW>>;
|
|
50
|
+
createTable(_table: string, _schema: JsonSchemaObject, opt?: CommonDBCreateOptions): Promise<void>;
|
|
52
51
|
getByIds<ROW extends ObjectWithId>(_table: string, ids: string[], _opt?: CommonDBOptions): Promise<ROW[]>;
|
|
53
52
|
saveBatch<ROW extends ObjectWithId>(_table: string, rows: ROW[], _opt?: CommonDBSaveOptions): Promise<void>;
|
|
54
53
|
deleteByIds(_table: string, ids: string[], _opt?: CommonDBOptions): Promise<number>;
|
|
@@ -8,8 +8,7 @@ const fs = require("fs-extra");
|
|
|
8
8
|
const stream_1 = require("stream");
|
|
9
9
|
const zlib_1 = require("zlib");
|
|
10
10
|
const __1 = require("../..");
|
|
11
|
-
const
|
|
12
|
-
const log = nodejs_lib_1.Debug('nc:db-lib:inmemorydb');
|
|
11
|
+
const log = (0, nodejs_lib_1.Debug)('nc:db-lib:inmemorydb');
|
|
13
12
|
class InMemoryDB {
|
|
14
13
|
constructor(cfg) {
|
|
15
14
|
// data[table][id] > {id: 'a', created: ... }
|
|
@@ -28,7 +27,7 @@ class InMemoryDB {
|
|
|
28
27
|
* Deterministic - jsonSorted.
|
|
29
28
|
*/
|
|
30
29
|
getDataSnapshot() {
|
|
31
|
-
return js_lib_1._sortObjectDeep(this.data);
|
|
30
|
+
return (0, js_lib_1._sortObjectDeep)(this.data);
|
|
32
31
|
}
|
|
33
32
|
async ping() { }
|
|
34
33
|
/**
|
|
@@ -53,10 +52,13 @@ class InMemoryDB {
|
|
|
53
52
|
}
|
|
54
53
|
async getTableSchema(_table) {
|
|
55
54
|
const table = this.cfg.tablesPrefix + _table;
|
|
56
|
-
return
|
|
55
|
+
return {
|
|
56
|
+
...(0, js_lib_1.generateJsonSchemaFromData)(Object.values(this.data[table] || {})),
|
|
57
|
+
$id: `${table}.schema.json`,
|
|
58
|
+
};
|
|
57
59
|
}
|
|
58
|
-
async createTable(
|
|
59
|
-
const table = this.cfg.tablesPrefix +
|
|
60
|
+
async createTable(_table, _schema, opt = {}) {
|
|
61
|
+
const table = this.cfg.tablesPrefix + _table;
|
|
60
62
|
if (opt.dropIfExists) {
|
|
61
63
|
this.data[table] = {};
|
|
62
64
|
}
|
|
@@ -103,21 +105,21 @@ class InMemoryDB {
|
|
|
103
105
|
}
|
|
104
106
|
async deleteByQuery(q, _opt) {
|
|
105
107
|
const table = this.cfg.tablesPrefix + q.table;
|
|
106
|
-
const rows = __1.queryInMemory(q, Object.values(this.data[table] || {}));
|
|
108
|
+
const rows = (0, __1.queryInMemory)(q, Object.values(this.data[table] || {}));
|
|
107
109
|
const ids = rows.map(r => r.id);
|
|
108
110
|
return await this.deleteByIds(q.table, ids);
|
|
109
111
|
}
|
|
110
112
|
async runQuery(q, _opt) {
|
|
111
113
|
const table = this.cfg.tablesPrefix + q.table;
|
|
112
|
-
return { rows: __1.queryInMemory(q, Object.values(this.data[table] || {})) };
|
|
114
|
+
return { rows: (0, __1.queryInMemory)(q, Object.values(this.data[table] || {})) };
|
|
113
115
|
}
|
|
114
116
|
async runQueryCount(q, _opt) {
|
|
115
117
|
const table = this.cfg.tablesPrefix + q.table;
|
|
116
|
-
return __1.queryInMemory(q, Object.values(this.data[table] || {})).length;
|
|
118
|
+
return (0, __1.queryInMemory)(q, Object.values(this.data[table] || {})).length;
|
|
117
119
|
}
|
|
118
120
|
streamQuery(q, _opt) {
|
|
119
121
|
const table = this.cfg.tablesPrefix + q.table;
|
|
120
|
-
return stream_1.Readable.from(__1.queryInMemory(q, Object.values(this.data[table] || {})));
|
|
122
|
+
return stream_1.Readable.from((0, __1.queryInMemory)(q, Object.values(this.data[table] || {})));
|
|
121
123
|
}
|
|
122
124
|
async commitTransaction(tx, opt) {
|
|
123
125
|
for await (const op of tx.ops) {
|
|
@@ -142,23 +144,23 @@ class InMemoryDB {
|
|
|
142
144
|
const { persistentStoragePath, persistZip } = this.cfg;
|
|
143
145
|
const started = Date.now();
|
|
144
146
|
await fs.emptyDir(persistentStoragePath);
|
|
145
|
-
const transformZip = persistZip ? [zlib_1.createGzip()] : [];
|
|
147
|
+
const transformZip = persistZip ? [(0, zlib_1.createGzip)()] : [];
|
|
146
148
|
let tables = 0;
|
|
147
149
|
// infinite concurrency for now
|
|
148
|
-
await js_lib_1.pMap(Object.keys(this.data), async (table) => {
|
|
150
|
+
await (0, js_lib_1.pMap)(Object.keys(this.data), async (table) => {
|
|
149
151
|
const rows = Object.values(this.data[table]);
|
|
150
152
|
if (rows.length === 0)
|
|
151
153
|
return; // 0 rows
|
|
152
154
|
tables++;
|
|
153
155
|
const fname = `${persistentStoragePath}/${table}.ndjson${persistZip ? '.gz' : ''}`;
|
|
154
|
-
await nodejs_lib_1._pipeline([
|
|
156
|
+
await (0, nodejs_lib_1._pipeline)([
|
|
155
157
|
stream_1.Readable.from(rows),
|
|
156
|
-
nodejs_lib_1.transformToNDJson(),
|
|
158
|
+
(0, nodejs_lib_1.transformToNDJson)(),
|
|
157
159
|
...transformZip,
|
|
158
160
|
fs.createWriteStream(fname),
|
|
159
161
|
]);
|
|
160
162
|
});
|
|
161
|
-
log(`flushToDisk took ${colors_1.dimGrey(js_lib_1._since(started))} to save ${colors_1.yellow(tables)} tables`);
|
|
163
|
+
log(`flushToDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to save ${(0, colors_1.yellow)(tables)} tables`);
|
|
162
164
|
}
|
|
163
165
|
/**
|
|
164
166
|
* Restores all tables (all namespaces) at once.
|
|
@@ -173,21 +175,21 @@ class InMemoryDB {
|
|
|
173
175
|
this.data = {}; // empty it in the beginning!
|
|
174
176
|
const files = (await fs.readdir(persistentStoragePath)).filter(f => f.includes('.ndjson'));
|
|
175
177
|
// infinite concurrency for now
|
|
176
|
-
await js_lib_1.pMap(files, async (file) => {
|
|
178
|
+
await (0, js_lib_1.pMap)(files, async (file) => {
|
|
177
179
|
const fname = `${persistentStoragePath}/${file}`;
|
|
178
180
|
const table = file.split('.ndjson')[0];
|
|
179
|
-
const transformUnzip = file.endsWith('.gz') ? [zlib_1.createUnzip()] : [];
|
|
181
|
+
const transformUnzip = file.endsWith('.gz') ? [(0, zlib_1.createUnzip)()] : [];
|
|
180
182
|
const rows = [];
|
|
181
|
-
await nodejs_lib_1._pipeline([
|
|
183
|
+
await (0, nodejs_lib_1._pipeline)([
|
|
182
184
|
fs.createReadStream(fname),
|
|
183
185
|
...transformUnzip,
|
|
184
|
-
nodejs_lib_1.transformSplit(),
|
|
185
|
-
nodejs_lib_1.transformJsonParse(),
|
|
186
|
-
nodejs_lib_1.writablePushToArray(rows),
|
|
186
|
+
(0, nodejs_lib_1.transformSplit)(),
|
|
187
|
+
(0, nodejs_lib_1.transformJsonParse)(),
|
|
188
|
+
(0, nodejs_lib_1.writablePushToArray)(rows),
|
|
187
189
|
]);
|
|
188
|
-
this.data[table] = js_lib_1._by(rows, r => r.id);
|
|
190
|
+
this.data[table] = (0, js_lib_1._by)(rows, r => r.id);
|
|
189
191
|
});
|
|
190
|
-
log(`restoreFromDisk took ${colors_1.dimGrey(js_lib_1._since(started))} to read ${colors_1.yellow(files.length)} tables`);
|
|
192
|
+
log(`restoreFromDisk took ${(0, colors_1.dimGrey)((0, js_lib_1._since)(started))} to read ${(0, colors_1.yellow)(files.length)} tables`);
|
|
191
193
|
}
|
|
192
194
|
}
|
|
193
195
|
exports.InMemoryDB = InMemoryDB;
|
|
@@ -23,7 +23,7 @@ function queryInMemory(q, rows = []) {
|
|
|
23
23
|
}, rows);
|
|
24
24
|
// .select(fieldNames)
|
|
25
25
|
if (q._selectedFieldNames) {
|
|
26
|
-
rows = rows.map(row => js_lib_1._pick(row, q._selectedFieldNames));
|
|
26
|
+
rows = rows.map(row => (0, js_lib_1._pick)(row, q._selectedFieldNames));
|
|
27
27
|
}
|
|
28
28
|
// todo: only one order is supported (first)
|
|
29
29
|
const [order] = q._orders;
|
package/dist/base.common.db.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import { JsonSchemaObject } from '@naturalcycles/js-lib';
|
|
1
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
2
3
|
import { CommonDB } from './common.db';
|
|
3
4
|
import { CommonDBSaveOptions, ObjectWithId, RunQueryResult } from './db.model';
|
|
4
5
|
import { DBQuery } from './query/dbQuery';
|
|
5
|
-
import { CommonSchema } from './schema/common.schema';
|
|
6
6
|
import { DBTransaction } from './transaction/dbTransaction';
|
|
7
7
|
/**
|
|
8
8
|
* No-op implementation of CommonDB interface.
|
|
@@ -11,8 +11,8 @@ import { DBTransaction } from './transaction/dbTransaction';
|
|
|
11
11
|
export declare class BaseCommonDB implements CommonDB {
|
|
12
12
|
ping(): Promise<void>;
|
|
13
13
|
getTables(): Promise<string[]>;
|
|
14
|
-
getTableSchema<ROW>(table: string): Promise<
|
|
15
|
-
createTable(_schema:
|
|
14
|
+
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaObject<ROW>>;
|
|
15
|
+
createTable(_table: string, _schema: JsonSchemaObject): Promise<void>;
|
|
16
16
|
deleteByIds(_table: string, _ids: string[]): Promise<number>;
|
|
17
17
|
deleteByQuery<ROW extends ObjectWithId>(_q: DBQuery<ROW>): Promise<number>;
|
|
18
18
|
getByIds<ROW extends ObjectWithId>(_table: string, _ids: string[]): Promise<ROW[]>;
|
package/dist/base.common.db.js
CHANGED
|
@@ -13,9 +13,15 @@ class BaseCommonDB {
|
|
|
13
13
|
return [];
|
|
14
14
|
}
|
|
15
15
|
async getTableSchema(table) {
|
|
16
|
-
return {
|
|
17
|
-
|
|
18
|
-
|
|
16
|
+
return {
|
|
17
|
+
$id: `${table}.schema.json`,
|
|
18
|
+
type: 'object',
|
|
19
|
+
additionalProperties: true,
|
|
20
|
+
properties: {},
|
|
21
|
+
required: [],
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
async createTable(_table, _schema) { }
|
|
19
25
|
async deleteByIds(_table, _ids) {
|
|
20
26
|
return 0;
|
|
21
27
|
}
|
|
@@ -40,7 +46,7 @@ class BaseCommonDB {
|
|
|
40
46
|
* To be extended.
|
|
41
47
|
*/
|
|
42
48
|
async commitTransaction(tx, opt) {
|
|
43
|
-
await dbTransaction_util_1.commitDBTransactionSimple(this, tx, opt);
|
|
49
|
+
await (0, dbTransaction_util_1.commitDBTransactionSimple)(this, tx, opt);
|
|
44
50
|
}
|
|
45
51
|
}
|
|
46
52
|
exports.BaseCommonDB = BaseCommonDB;
|
package/dist/common.db.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
+
import { JsonSchemaObject } from '@naturalcycles/js-lib';
|
|
1
2
|
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
2
3
|
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, ObjectWithId, RunQueryResult } from './db.model';
|
|
3
4
|
import { DBQuery } from './query/dbQuery';
|
|
4
|
-
import { CommonSchema } from './schema/common.schema';
|
|
5
5
|
import { DBTransaction } from './transaction/dbTransaction';
|
|
6
6
|
export interface CommonDB {
|
|
7
7
|
/**
|
|
@@ -15,12 +15,12 @@ export interface CommonDB {
|
|
|
15
15
|
* Return all tables (table names) available in this DB.
|
|
16
16
|
*/
|
|
17
17
|
getTables(): Promise<string[]>;
|
|
18
|
-
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<
|
|
18
|
+
getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaObject<ROW>>;
|
|
19
19
|
/**
|
|
20
20
|
* Will do like `create table ...` for mysql.
|
|
21
21
|
* Caution! dropIfExists defaults to false. If set to true - will actually DROP the table!
|
|
22
22
|
*/
|
|
23
|
-
createTable(schema:
|
|
23
|
+
createTable(table: string, schema: JsonSchemaObject, opt?: CommonDBCreateOptions): Promise<void>;
|
|
24
24
|
/**
|
|
25
25
|
* Order of items returned is not guaranteed to match order of ids.
|
|
26
26
|
* (Such limitation exists because Datastore doesn't support it).
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { AsyncMapper } from '@naturalcycles/js-lib';
|
|
2
|
-
import { ObjectSchemaTyped, ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
1
|
+
import { AsyncMapper, JsonSchemaObject } from '@naturalcycles/js-lib';
|
|
2
|
+
import { AjvSchema, ObjectSchemaTyped, ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
3
|
import { DBModelType, ObjectWithId, RunQueryResult, Saved } from '../db.model';
|
|
4
4
|
import { DBQuery, RunnableDBQuery } from '../query/dbQuery';
|
|
5
|
-
import { CommonSchema } from '../schema/common.schema';
|
|
6
5
|
import { CommonDaoCfg, CommonDaoCreateOptions, CommonDaoOptions, CommonDaoSaveOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions } from './common.dao.model';
|
|
7
6
|
/**
|
|
8
7
|
* Lowest common denominator API between supported Databases.
|
|
@@ -116,11 +115,12 @@ export declare class CommonDao<BM extends Partial<ObjectWithId>, DBM extends Obj
|
|
|
116
115
|
/**
|
|
117
116
|
* Returns *converted value*.
|
|
118
117
|
* Validates (unless `skipValidation=true` passed).
|
|
119
|
-
*
|
|
118
|
+
*
|
|
119
|
+
* Does NOT mutate the object.
|
|
120
120
|
*/
|
|
121
|
-
validateAndConvert<IN
|
|
122
|
-
getTableSchema(): Promise<
|
|
123
|
-
createTable(schema:
|
|
121
|
+
validateAndConvert<IN, OUT = IN>(obj: IN, schema?: ObjectSchemaTyped<IN> | AjvSchema<IN>, modelType?: DBModelType, opt?: CommonDaoOptions): OUT;
|
|
122
|
+
getTableSchema(): Promise<JsonSchemaObject<DBM>>;
|
|
123
|
+
createTable(schema: JsonSchemaObject<DBM>, opt?: CommonDaoCreateOptions): Promise<void>;
|
|
124
124
|
/**
|
|
125
125
|
* Proxy to this.cfg.db.ping
|
|
126
126
|
*/
|