@naturalcycles/db-lib 10.0.1 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/cachedb/cache.db.d.ts +7 -7
- package/dist/adapter/cachedb/cache.db.js +13 -16
- package/dist/adapter/cachedb/cache.db.model.d.ts +3 -3
- package/dist/adapter/cachedb/cache.db.model.js +1 -2
- package/dist/adapter/cachedb/index.d.ts +2 -2
- package/dist/adapter/cachedb/index.js +2 -5
- package/dist/adapter/file/file.db.d.ts +7 -7
- package/dist/adapter/file/file.db.js +35 -36
- package/dist/adapter/file/file.db.model.d.ts +3 -3
- package/dist/adapter/file/file.db.model.js +1 -2
- package/dist/adapter/file/inMemory.persistence.plugin.d.ts +3 -3
- package/dist/adapter/file/inMemory.persistence.plugin.js +4 -10
- package/dist/adapter/file/index.d.ts +3 -3
- package/dist/adapter/file/index.js +3 -7
- package/dist/adapter/file/localFile.persistence.plugin.d.ts +3 -3
- package/dist/adapter/file/localFile.persistence.plugin.js +12 -15
- package/dist/adapter/file/noop.persistence.plugin.d.ts +3 -3
- package/dist/adapter/file/noop.persistence.plugin.js +1 -5
- package/dist/adapter/inmemory/inMemory.db.d.ts +7 -5
- package/dist/adapter/inmemory/inMemory.db.js +54 -49
- package/dist/adapter/inmemory/inMemoryKeyValueDB.d.ts +4 -4
- package/dist/adapter/inmemory/inMemoryKeyValueDB.js +12 -15
- package/dist/adapter/inmemory/queryInMemory.d.ts +2 -2
- package/dist/adapter/inmemory/queryInMemory.js +4 -7
- package/dist/base.common.db.d.ts +8 -5
- package/dist/base.common.db.js +9 -12
- package/dist/cnst.js +2 -5
- package/dist/common.db.d.ts +9 -3
- package/dist/common.db.js +3 -6
- package/dist/commondao/common.dao.d.ts +16 -6
- package/dist/commondao/common.dao.js +123 -112
- package/dist/commondao/common.dao.model.d.ts +5 -4
- package/dist/commondao/common.dao.model.js +2 -5
- package/dist/db.model.d.ts +7 -2
- package/dist/db.model.js +4 -7
- package/dist/index.d.ts +17 -17
- package/dist/index.js +17 -20
- package/dist/kv/commonKeyValueDB.d.ts +3 -3
- package/dist/kv/commonKeyValueDB.js +1 -4
- package/dist/kv/commonKeyValueDao.d.ts +5 -5
- package/dist/kv/commonKeyValueDao.js +11 -14
- package/dist/kv/commonKeyValueDaoMemoCache.d.ts +3 -2
- package/dist/kv/commonKeyValueDaoMemoCache.js +5 -8
- package/dist/model.util.d.ts +1 -1
- package/dist/model.util.js +9 -15
- package/dist/pipeline/dbPipelineBackup.d.ts +6 -4
- package/dist/pipeline/dbPipelineBackup.js +27 -30
- package/dist/pipeline/dbPipelineCopy.d.ts +6 -4
- package/dist/pipeline/dbPipelineCopy.js +20 -23
- package/dist/pipeline/dbPipelineRestore.d.ts +6 -4
- package/dist/pipeline/dbPipelineRestore.js +27 -30
- package/dist/query/dbQuery.d.ts +5 -5
- package/dist/query/dbQuery.js +22 -17
- package/dist/testing/{dbTest.d.ts → commonDBTest.d.ts} +1 -1
- package/dist/testing/{dbTest.js → commonDBTest.js} +81 -62
- package/dist/testing/commonDaoTest.d.ts +3 -0
- package/dist/testing/{daoTest.js → commonDaoTest.js} +101 -38
- package/dist/testing/index.d.ts +7 -5
- package/dist/testing/index.js +6 -20
- package/dist/testing/keyValueDBTest.d.ts +1 -1
- package/dist/testing/keyValueDBTest.js +28 -31
- package/dist/testing/keyValueDaoTest.d.ts +1 -1
- package/dist/testing/keyValueDaoTest.js +8 -11
- package/dist/testing/test.model.d.ts +1 -1
- package/dist/testing/test.model.js +30 -37
- package/dist/testing/timeSeriesTest.util.d.ts +1 -1
- package/dist/testing/timeSeriesTest.util.js +3 -6
- package/dist/timeseries/commonTimeSeriesDao.d.ts +1 -1
- package/dist/timeseries/commonTimeSeriesDao.js +6 -9
- package/dist/timeseries/timeSeries.model.d.ts +1 -1
- package/dist/timeseries/timeSeries.model.js +1 -2
- package/dist/transaction/dbTransaction.util.d.ts +4 -3
- package/dist/transaction/dbTransaction.util.js +3 -5
- package/dist/validation/index.d.ts +2 -2
- package/dist/validation/index.js +25 -28
- package/package.json +5 -4
- package/src/adapter/cachedb/cache.db.model.ts +4 -4
- package/src/adapter/cachedb/cache.db.ts +10 -9
- package/src/adapter/cachedb/index.ts +3 -3
- package/src/adapter/file/file.db.model.ts +3 -3
- package/src/adapter/file/file.db.ts +12 -16
- package/src/adapter/file/inMemory.persistence.plugin.ts +4 -3
- package/src/adapter/file/index.ts +3 -3
- package/src/adapter/file/localFile.persistence.plugin.ts +4 -3
- package/src/adapter/file/noop.persistence.plugin.ts +3 -3
- package/src/adapter/inmemory/inMemory.db.ts +26 -27
- package/src/adapter/inmemory/inMemoryKeyValueDB.ts +6 -6
- package/src/adapter/inmemory/queryInMemory.ts +3 -2
- package/src/base.common.db.ts +12 -7
- package/src/common.db.ts +12 -4
- package/src/commondao/common.dao.model.ts +5 -6
- package/src/commondao/common.dao.ts +41 -23
- package/src/db.model.ts +8 -2
- package/src/index.ts +17 -17
- package/src/kv/commonKeyValueDB.ts +3 -3
- package/src/kv/commonKeyValueDao.ts +8 -6
- package/src/kv/commonKeyValueDaoMemoCache.ts +3 -8
- package/src/model.util.ts +2 -1
- package/src/pipeline/dbPipelineBackup.ts +5 -14
- package/src/pipeline/dbPipelineCopy.ts +6 -14
- package/src/pipeline/dbPipelineRestore.ts +9 -8
- package/src/query/dbQuery.ts +7 -12
- package/src/testing/{dbTest.ts → commonDBTest.ts} +34 -5
- package/src/testing/{daoTest.ts → commonDaoTest.ts} +89 -11
- package/src/testing/index.ts +7 -8
- package/src/testing/keyValueDBTest.ts +4 -3
- package/src/testing/keyValueDaoTest.ts +3 -3
- package/src/testing/test.model.ts +2 -7
- package/src/testing/timeSeriesTest.util.ts +1 -1
- package/src/timeseries/commonTimeSeriesDao.ts +5 -4
- package/src/timeseries/timeSeries.model.ts +1 -1
- package/src/transaction/dbTransaction.util.ts +4 -3
- package/src/validation/index.ts +4 -4
- package/dist/testing/daoTest.d.ts +0 -3
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
-
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
-
import { BaseCommonDB } from '../../base.common.db';
|
|
4
|
-
import { CommonDB, CommonDBSupport } from '../../common.db';
|
|
5
|
-
import { RunQueryResult } from '../../db.model';
|
|
6
|
-
import { DBQuery } from '../../query/dbQuery';
|
|
7
|
-
import { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBSaveOptions, CacheDBStreamOptions } from './cache.db.model';
|
|
1
|
+
import type { JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
+
import { BaseCommonDB } from '../../base.common.db.js';
|
|
4
|
+
import type { CommonDB, CommonDBSupport } from '../../common.db.js';
|
|
5
|
+
import type { RunQueryResult } from '../../db.model.js';
|
|
6
|
+
import type { DBQuery } from '../../query/dbQuery.js';
|
|
7
|
+
import type { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBSaveOptions, CacheDBStreamOptions } from './cache.db.model.js';
|
|
8
8
|
/**
|
|
9
9
|
* CommonDB implementation that proxies requests to downstream CommonDB
|
|
10
10
|
* and does in-memory caching.
|
|
@@ -1,29 +1,27 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
|
-
const base_common_db_1 = require("../../base.common.db");
|
|
7
|
-
const common_db_1 = require("../../common.db");
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { _isTruthy } from '@naturalcycles/js-lib';
|
|
3
|
+
import { BaseCommonDB } from '../../base.common.db.js';
|
|
4
|
+
import { commonDBFullSupport } from '../../common.db.js';
|
|
8
5
|
/**
|
|
9
6
|
* CommonDB implementation that proxies requests to downstream CommonDB
|
|
10
7
|
* and does in-memory caching.
|
|
11
8
|
*
|
|
12
9
|
* Queries always hit downstream (unless `onlyCache` is passed)
|
|
13
10
|
*/
|
|
14
|
-
class CacheDB extends
|
|
11
|
+
export class CacheDB extends BaseCommonDB {
|
|
12
|
+
support = {
|
|
13
|
+
...commonDBFullSupport,
|
|
14
|
+
transactions: false,
|
|
15
|
+
increment: false,
|
|
16
|
+
};
|
|
15
17
|
constructor(cfg) {
|
|
16
18
|
super();
|
|
17
|
-
this.support = {
|
|
18
|
-
...common_db_1.commonDBFullSupport,
|
|
19
|
-
transactions: false,
|
|
20
|
-
increment: false,
|
|
21
|
-
};
|
|
22
19
|
this.cfg = {
|
|
23
20
|
logger: console,
|
|
24
21
|
...cfg,
|
|
25
22
|
};
|
|
26
23
|
}
|
|
24
|
+
cfg;
|
|
27
25
|
async ping() {
|
|
28
26
|
await Promise.all([this.cfg.cacheDB.ping(), this.cfg.downstreamDB.ping()]);
|
|
29
27
|
}
|
|
@@ -77,7 +75,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
|
|
|
77
75
|
}
|
|
78
76
|
}
|
|
79
77
|
// return in right order
|
|
80
|
-
return ids.map(id => resultMap[id]).filter(
|
|
78
|
+
return ids.map(id => resultMap[id]).filter(_isTruthy);
|
|
81
79
|
}
|
|
82
80
|
async saveBatch(table, rows, opt = {}) {
|
|
83
81
|
if (!opt.onlyCache && !this.cfg.onlyCache) {
|
|
@@ -146,7 +144,7 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
|
|
|
146
144
|
return stream;
|
|
147
145
|
}
|
|
148
146
|
if (opt.skipCache || this.cfg.skipCache)
|
|
149
|
-
return
|
|
147
|
+
return Readable.from([]);
|
|
150
148
|
const stream = this.cfg.cacheDB.streamQuery(q, opt);
|
|
151
149
|
// if (this.cfg.logCached) {
|
|
152
150
|
// let count = 0
|
|
@@ -194,4 +192,3 @@ class CacheDB extends base_common_db_1.BaseCommonDB {
|
|
|
194
192
|
return updated || 0;
|
|
195
193
|
}
|
|
196
194
|
}
|
|
197
|
-
exports.CacheDB = CacheDB;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
-
import { CommonDB } from '../../common.db';
|
|
3
|
-
import { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions } from '../../db.model';
|
|
1
|
+
import type { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { CommonDB } from '../../common.db.js';
|
|
3
|
+
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions } from '../../db.model.js';
|
|
4
4
|
export interface CacheDBCfg {
|
|
5
5
|
name: string;
|
|
6
6
|
cacheDB: CommonDB;
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CacheDB } from './cache.db';
|
|
2
|
-
import { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBStreamOptions } from './cache.db.model';
|
|
1
|
+
import { CacheDB } from './cache.db.js';
|
|
2
|
+
import type { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBStreamOptions } from './cache.db.model.js';
|
|
3
3
|
export type { CacheDBCfg, CacheDBCreateOptions, CacheDBOptions, CacheDBStreamOptions };
|
|
4
4
|
export { CacheDB };
|
|
@@ -1,5 +1,2 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.CacheDB = void 0;
|
|
4
|
-
const cache_db_1 = require("./cache.db");
|
|
5
|
-
Object.defineProperty(exports, "CacheDB", { enumerable: true, get: function () { return cache_db_1.CacheDB; } });
|
|
1
|
+
import { CacheDB } from './cache.db.js';
|
|
2
|
+
export { CacheDB };
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
-
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
-
import { BaseCommonDB
|
|
4
|
-
import { CommonDB } from '../../common.db';
|
|
5
|
-
import { CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, RunQueryResult } from '../../db.model';
|
|
6
|
-
import { DBQuery } from '../../query/dbQuery';
|
|
7
|
-
import { FileDBCfg } from './file.db.model';
|
|
1
|
+
import type { JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
+
import { BaseCommonDB } from '../../base.common.db.js';
|
|
4
|
+
import type { CommonDB, CommonDBSupport } from '../../common.db.js';
|
|
5
|
+
import type { CommonDBOptions, CommonDBSaveOptions, CommonDBStreamOptions, DBSaveBatchOperation, RunQueryResult } from '../../db.model.js';
|
|
6
|
+
import type { DBQuery } from '../../query/dbQuery.js';
|
|
7
|
+
import type { FileDBCfg } from './file.db.model.js';
|
|
8
8
|
/**
|
|
9
9
|
* Provides barebone implementation for "whole file" based CommonDB.
|
|
10
10
|
* "whole file" means that the persistence layer doesn't allow any querying,
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
const __1 = require("../..");
|
|
1
|
+
import { _assert, _by, _deepEquals, _filterUndefinedValues, _since, _sortBy, _sortObjectDeep, _stringMapValues, generateJsonSchemaFromData, localTime, } from '@naturalcycles/js-lib';
|
|
2
|
+
import { dimGrey, readableCreate } from '@naturalcycles/nodejs-lib';
|
|
3
|
+
import { BaseCommonDB } from '../../base.common.db.js';
|
|
4
|
+
import { commonDBFullSupport } from '../../common.db.js';
|
|
5
|
+
import { queryInMemory } from '../inmemory/queryInMemory.js';
|
|
7
6
|
/**
|
|
8
7
|
* Provides barebone implementation for "whole file" based CommonDB.
|
|
9
8
|
* "whole file" means that the persistence layer doesn't allow any querying,
|
|
@@ -14,19 +13,19 @@ const __1 = require("../..");
|
|
|
14
13
|
* Each load/query operation loads *whole* file from the persitence layer.
|
|
15
14
|
* Each save operation saves *whole* file to the persistence layer.
|
|
16
15
|
*/
|
|
17
|
-
class FileDB extends
|
|
16
|
+
export class FileDB extends BaseCommonDB {
|
|
17
|
+
support = {
|
|
18
|
+
...commonDBFullSupport,
|
|
19
|
+
bufferValues: false, // todo: implement
|
|
20
|
+
insertSaveMethod: false,
|
|
21
|
+
updateSaveMethod: false,
|
|
22
|
+
patchByQuery: false,
|
|
23
|
+
createTable: false,
|
|
24
|
+
transactions: false, // todo
|
|
25
|
+
increment: false,
|
|
26
|
+
};
|
|
18
27
|
constructor(cfg) {
|
|
19
28
|
super();
|
|
20
|
-
this.support = {
|
|
21
|
-
...__1.commonDBFullSupport,
|
|
22
|
-
bufferValues: false, // todo: implement
|
|
23
|
-
insertSaveMethod: false,
|
|
24
|
-
updateSaveMethod: false,
|
|
25
|
-
patchByQuery: false,
|
|
26
|
-
createTable: false,
|
|
27
|
-
transactions: false, // todo
|
|
28
|
-
increment: false,
|
|
29
|
-
};
|
|
30
29
|
this.cfg = {
|
|
31
30
|
sortObjects: true,
|
|
32
31
|
logFinished: true,
|
|
@@ -34,6 +33,7 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
34
33
|
...cfg,
|
|
35
34
|
};
|
|
36
35
|
}
|
|
36
|
+
cfg;
|
|
37
37
|
async ping() {
|
|
38
38
|
await this.cfg.plugin.ping();
|
|
39
39
|
}
|
|
@@ -44,19 +44,19 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
44
44
|
return tables;
|
|
45
45
|
}
|
|
46
46
|
async getByIds(table, ids, _opt) {
|
|
47
|
-
const byId =
|
|
47
|
+
const byId = _by(await this.loadFile(table), r => r.id);
|
|
48
48
|
return ids.map(id => byId[id]).filter(Boolean);
|
|
49
49
|
}
|
|
50
50
|
async saveBatch(table, rows, _opt) {
|
|
51
51
|
if (!rows.length)
|
|
52
52
|
return; // save some api calls
|
|
53
53
|
// 1. Load the whole file
|
|
54
|
-
const byId =
|
|
54
|
+
const byId = _by(await this.loadFile(table), r => r.id);
|
|
55
55
|
// 2. Merge with new data (using ids)
|
|
56
56
|
let saved = 0;
|
|
57
57
|
rows.forEach(r => {
|
|
58
|
-
|
|
59
|
-
if (!
|
|
58
|
+
_assert(r.id, 'FileDB: row.id is required');
|
|
59
|
+
if (!_deepEquals(byId[r.id], r)) {
|
|
60
60
|
byId[r.id] = r;
|
|
61
61
|
saved++;
|
|
62
62
|
}
|
|
@@ -64,19 +64,19 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
64
64
|
// Only save if there are changed rows
|
|
65
65
|
if (saved > 0) {
|
|
66
66
|
// 3. Save the whole file
|
|
67
|
-
await this.saveFile(table,
|
|
67
|
+
await this.saveFile(table, _stringMapValues(byId));
|
|
68
68
|
}
|
|
69
69
|
}
|
|
70
70
|
async runQuery(q, _opt) {
|
|
71
71
|
return {
|
|
72
|
-
rows:
|
|
72
|
+
rows: queryInMemory(q, await this.loadFile(q.table)),
|
|
73
73
|
};
|
|
74
74
|
}
|
|
75
75
|
async runQueryCount(q, _opt) {
|
|
76
76
|
return (await this.loadFile(q.table)).length;
|
|
77
77
|
}
|
|
78
78
|
streamQuery(q, opt) {
|
|
79
|
-
const readable =
|
|
79
|
+
const readable = readableCreate();
|
|
80
80
|
void this.runQuery(q, opt).then(({ rows }) => {
|
|
81
81
|
rows.forEach(r => readable.push(r));
|
|
82
82
|
readable.push(null); // done
|
|
@@ -84,19 +84,19 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
84
84
|
return readable;
|
|
85
85
|
}
|
|
86
86
|
async deleteByQuery(q, _opt) {
|
|
87
|
-
const byId =
|
|
87
|
+
const byId = _by(await this.loadFile(q.table), r => r.id);
|
|
88
88
|
let deleted = 0;
|
|
89
|
-
|
|
89
|
+
queryInMemory(q, _stringMapValues(byId)).forEach(r => {
|
|
90
90
|
delete byId[r.id];
|
|
91
91
|
deleted++;
|
|
92
92
|
});
|
|
93
93
|
if (deleted > 0) {
|
|
94
|
-
await this.saveFile(q.table,
|
|
94
|
+
await this.saveFile(q.table, _stringMapValues(byId));
|
|
95
95
|
}
|
|
96
96
|
return deleted;
|
|
97
97
|
}
|
|
98
98
|
async deleteByIds(table, ids, _opt) {
|
|
99
|
-
const byId =
|
|
99
|
+
const byId = _by(await this.loadFile(table), r => r.id);
|
|
100
100
|
let deleted = 0;
|
|
101
101
|
ids.forEach(id => {
|
|
102
102
|
if (!byId[id])
|
|
@@ -105,14 +105,14 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
105
105
|
deleted++;
|
|
106
106
|
});
|
|
107
107
|
if (deleted > 0) {
|
|
108
|
-
await this.saveFile(table,
|
|
108
|
+
await this.saveFile(table, _stringMapValues(byId));
|
|
109
109
|
}
|
|
110
110
|
return deleted;
|
|
111
111
|
}
|
|
112
112
|
async getTableSchema(table) {
|
|
113
113
|
const rows = await this.loadFile(table);
|
|
114
114
|
return {
|
|
115
|
-
...
|
|
115
|
+
...generateJsonSchemaFromData(rows),
|
|
116
116
|
$id: `${table}.schema.json`,
|
|
117
117
|
};
|
|
118
118
|
}
|
|
@@ -145,14 +145,14 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
145
145
|
// return new FileDBTransaction(this)
|
|
146
146
|
// }
|
|
147
147
|
sortRows(rows) {
|
|
148
|
-
rows = rows.map(r =>
|
|
148
|
+
rows = rows.map(r => _filterUndefinedValues(r));
|
|
149
149
|
if (this.cfg.sortOnSave) {
|
|
150
|
-
|
|
150
|
+
_sortBy(rows, r => r[this.cfg.sortOnSave.name], true);
|
|
151
151
|
if (this.cfg.sortOnSave.descending)
|
|
152
152
|
rows.reverse(); // mutates
|
|
153
153
|
}
|
|
154
154
|
if (this.cfg.sortObjects) {
|
|
155
|
-
return
|
|
155
|
+
return _sortObjectDeep(rows);
|
|
156
156
|
}
|
|
157
157
|
return rows;
|
|
158
158
|
}
|
|
@@ -160,15 +160,14 @@ class FileDB extends __1.BaseCommonDB {
|
|
|
160
160
|
if (this.cfg.logStarted) {
|
|
161
161
|
this.cfg.logger?.log(`>> ${op}`);
|
|
162
162
|
}
|
|
163
|
-
return
|
|
163
|
+
return localTime.nowUnixMillis();
|
|
164
164
|
}
|
|
165
165
|
logFinished(started, op) {
|
|
166
166
|
if (!this.cfg.logFinished)
|
|
167
167
|
return;
|
|
168
|
-
this.cfg.logger?.log(`<< ${op} ${
|
|
168
|
+
this.cfg.logger?.log(`<< ${op} ${dimGrey(`in ${_since(started)}`)}`);
|
|
169
169
|
}
|
|
170
170
|
}
|
|
171
|
-
exports.FileDB = FileDB;
|
|
172
171
|
// todo: get back and fix it
|
|
173
172
|
// Implementation is optimized for loading/saving _whole files_.
|
|
174
173
|
/*
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
-
import { DBSaveBatchOperation } from '../../db.model';
|
|
3
|
-
import type { DBQueryOrder } from '../../query/dbQuery';
|
|
1
|
+
import type { CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { DBSaveBatchOperation } from '../../db.model.js';
|
|
3
|
+
import type { DBQueryOrder } from '../../query/dbQuery.js';
|
|
4
4
|
export interface FileDBPersistencePlugin {
|
|
5
5
|
ping: () => Promise<void>;
|
|
6
6
|
getTables: () => Promise<string[]>;
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { ObjectWithId, StringMap } from '@naturalcycles/js-lib';
|
|
2
|
-
import { DBSaveBatchOperation } from '../../db.model';
|
|
3
|
-
import { FileDBPersistencePlugin } from './file.db.model';
|
|
1
|
+
import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { DBSaveBatchOperation } from '../../db.model.js';
|
|
3
|
+
import type { FileDBPersistencePlugin } from './file.db.model.js';
|
|
4
4
|
/**
|
|
5
5
|
* Mostly useful for testing.
|
|
6
6
|
*/
|
|
@@ -1,14 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.InMemoryPersistencePlugin = void 0;
|
|
4
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
1
|
+
import { _by } from '@naturalcycles/js-lib';
|
|
5
2
|
/**
|
|
6
3
|
* Mostly useful for testing.
|
|
7
4
|
*/
|
|
8
|
-
class InMemoryPersistencePlugin {
|
|
9
|
-
|
|
10
|
-
this.data = {};
|
|
11
|
-
}
|
|
5
|
+
export class InMemoryPersistencePlugin {
|
|
6
|
+
data = {};
|
|
12
7
|
async ping() { }
|
|
13
8
|
async getTables() {
|
|
14
9
|
return Object.keys(this.data);
|
|
@@ -18,8 +13,7 @@ class InMemoryPersistencePlugin {
|
|
|
18
13
|
}
|
|
19
14
|
async saveFiles(ops) {
|
|
20
15
|
ops.forEach(op => {
|
|
21
|
-
this.data[op.table] =
|
|
16
|
+
this.data[op.table] = _by(op.rows, r => r.id);
|
|
22
17
|
});
|
|
23
18
|
}
|
|
24
19
|
}
|
|
25
|
-
exports.InMemoryPersistencePlugin = InMemoryPersistencePlugin;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { FileDB } from './file.db';
|
|
2
|
-
import { FileDBCfg, FileDBPersistencePlugin } from './file.db.model';
|
|
3
|
-
import { LocalFilePersistencePlugin } from './localFile.persistence.plugin';
|
|
1
|
+
import { FileDB } from './file.db.js';
|
|
2
|
+
import type { FileDBCfg, FileDBPersistencePlugin } from './file.db.model.js';
|
|
3
|
+
import { LocalFilePersistencePlugin } from './localFile.persistence.plugin.js';
|
|
4
4
|
export type { FileDBCfg, FileDBPersistencePlugin };
|
|
5
5
|
export { FileDB, LocalFilePersistencePlugin };
|
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const file_db_1 = require("./file.db");
|
|
5
|
-
Object.defineProperty(exports, "FileDB", { enumerable: true, get: function () { return file_db_1.FileDB; } });
|
|
6
|
-
const localFile_persistence_plugin_1 = require("./localFile.persistence.plugin");
|
|
7
|
-
Object.defineProperty(exports, "LocalFilePersistencePlugin", { enumerable: true, get: function () { return localFile_persistence_plugin_1.LocalFilePersistencePlugin; } });
|
|
1
|
+
import { FileDB } from './file.db.js';
|
|
2
|
+
import { LocalFilePersistencePlugin } from './localFile.persistence.plugin.js';
|
|
3
|
+
export { FileDB, LocalFilePersistencePlugin };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
-
import { DBSaveBatchOperation } from '../../db.model';
|
|
3
|
-
import { FileDBPersistencePlugin } from './file.db.model';
|
|
1
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { DBSaveBatchOperation } from '../../db.model.js';
|
|
3
|
+
import type { FileDBPersistencePlugin } from './file.db.model.js';
|
|
4
4
|
export interface LocalFilePersistencePluginCfg {
|
|
5
5
|
/**
|
|
6
6
|
* @default ./tmp/localdb
|
|
@@ -1,13 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
const node_stream_1 = require("node:stream");
|
|
5
|
-
const js_lib_1 = require("@naturalcycles/js-lib");
|
|
6
|
-
const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
|
|
1
|
+
import { Readable } from 'node:stream';
|
|
2
|
+
import { pMap } from '@naturalcycles/js-lib';
|
|
3
|
+
import { _pipeline, fs2 } from '@naturalcycles/nodejs-lib';
|
|
7
4
|
/**
|
|
8
5
|
* Persists in local filesystem as ndjson.
|
|
9
6
|
*/
|
|
10
|
-
class LocalFilePersistencePlugin {
|
|
7
|
+
export class LocalFilePersistencePlugin {
|
|
11
8
|
constructor(cfg = {}) {
|
|
12
9
|
this.cfg = {
|
|
13
10
|
storagePath: './tmp/localdb',
|
|
@@ -15,28 +12,28 @@ class LocalFilePersistencePlugin {
|
|
|
15
12
|
...cfg,
|
|
16
13
|
};
|
|
17
14
|
}
|
|
15
|
+
cfg;
|
|
18
16
|
async ping() { }
|
|
19
17
|
async getTables() {
|
|
20
|
-
return (await
|
|
18
|
+
return (await fs2.readdirAsync(this.cfg.storagePath))
|
|
21
19
|
.filter(f => f.includes('.ndjson'))
|
|
22
20
|
.map(f => f.split('.ndjson')[0]);
|
|
23
21
|
}
|
|
24
22
|
async loadFile(table) {
|
|
25
|
-
await
|
|
23
|
+
await fs2.ensureDirAsync(this.cfg.storagePath);
|
|
26
24
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
|
|
27
25
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
|
|
28
|
-
if (!(await
|
|
26
|
+
if (!(await fs2.pathExistsAsync(filePath)))
|
|
29
27
|
return [];
|
|
30
|
-
return await
|
|
28
|
+
return await fs2.createReadStreamAsNDJSON(filePath).toArray();
|
|
31
29
|
}
|
|
32
30
|
async saveFiles(ops) {
|
|
33
|
-
await
|
|
31
|
+
await pMap(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 32 });
|
|
34
32
|
}
|
|
35
33
|
async saveFile(table, rows) {
|
|
36
|
-
await
|
|
34
|
+
await fs2.ensureDirAsync(this.cfg.storagePath);
|
|
37
35
|
const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
|
|
38
36
|
const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
|
|
39
|
-
await
|
|
37
|
+
await _pipeline([Readable.from(rows), ...fs2.createWriteStreamAsNDJSON(filePath)]);
|
|
40
38
|
}
|
|
41
39
|
}
|
|
42
|
-
exports.LocalFilePersistencePlugin = LocalFilePersistencePlugin;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
-
import { DBSaveBatchOperation } from '../../db.model';
|
|
3
|
-
import { FileDBPersistencePlugin } from './file.db.model';
|
|
1
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { DBSaveBatchOperation } from '../../db.model.js';
|
|
3
|
+
import type { FileDBPersistencePlugin } from './file.db.model.js';
|
|
4
4
|
export declare class NoopPersistencePlugin implements FileDBPersistencePlugin {
|
|
5
5
|
ping(): Promise<void>;
|
|
6
6
|
getTables(): Promise<string[]>;
|
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.NoopPersistencePlugin = void 0;
|
|
4
|
-
class NoopPersistencePlugin {
|
|
1
|
+
export class NoopPersistencePlugin {
|
|
5
2
|
async ping() { }
|
|
6
3
|
async getTables() {
|
|
7
4
|
return [];
|
|
@@ -11,4 +8,3 @@ class NoopPersistencePlugin {
|
|
|
11
8
|
}
|
|
12
9
|
async saveFiles(_ops) { }
|
|
13
10
|
}
|
|
14
|
-
exports.NoopPersistencePlugin = NoopPersistencePlugin;
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { AnyObjectWithId, CommonLogger, JsonSchemaObject, JsonSchemaRootObject, ObjectWithId, StringMap } from '@naturalcycles/js-lib';
|
|
2
|
-
import { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
-
import { CommonDB, CommonDBSupport
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
1
|
+
import type { AnyObjectWithId, CommonLogger, JsonSchemaObject, JsonSchemaRootObject, ObjectWithId, StringMap } from '@naturalcycles/js-lib';
|
|
2
|
+
import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
|
|
3
|
+
import type { CommonDB, CommonDBSupport } from '../../common.db.js';
|
|
4
|
+
import { CommonDBType } from '../../common.db.js';
|
|
5
|
+
import type { CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, CommonDBTransactionOptions, DBOperation, DBTransaction, DBTransactionFn, RunQueryResult } from '../../db.model.js';
|
|
6
|
+
import type { DBQuery } from '../../query/dbQuery.js';
|
|
6
7
|
export interface InMemoryDBCfg {
|
|
7
8
|
/**
|
|
8
9
|
* @default ''
|
|
@@ -73,6 +74,7 @@ export declare class InMemoryDB implements CommonDB {
|
|
|
73
74
|
runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
|
|
74
75
|
streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): ReadableTyped<ROW>;
|
|
75
76
|
runInTransaction(fn: DBTransactionFn, opt?: CommonDBTransactionOptions): Promise<void>;
|
|
77
|
+
createTransaction(opt?: CommonDBTransactionOptions): Promise<DBTransaction>;
|
|
76
78
|
incrementBatch(table: string, prop: string, incrementMap: StringMap<number>, _opt?: CommonDBOptions): Promise<StringMap<number>>;
|
|
77
79
|
/**
|
|
78
80
|
* Flushes all tables (all namespaces) at once.
|