@naturalcycles/db-lib 10.50.0 → 10.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/file/localFile.persistence.plugin.js +1 -1
- package/dist/cnst.js +1 -2
- package/dist/commondao/common.dao.js +9 -9
- package/dist/commondao/common.dao.model.js +1 -2
- package/dist/commondb/common.db.js +1 -2
- package/dist/db.model.js +2 -4
- package/dist/inmemory/inMemoryKeyValueDB.d.ts +2 -2
- package/dist/kv/index.d.ts +1 -1
- package/dist/kv/index.js +0 -1
- package/dist/pipeline/dbPipelineBackup.js +1 -1
- package/dist/pipeline/dbPipelineCopy.js +2 -2
- package/dist/pipeline/dbPipelineRestore.js +3 -3
- package/dist/testing/commonDBTest.js +3 -3
- package/dist/testing/commonDaoTest.js +4 -4
- package/dist/timeseries/commonTimeSeriesDao.js +1 -1
- package/dist/validation/index.d.ts +2 -0
- package/package.json +3 -3
- package/src/kv/index.ts +1 -1
|
@@ -28,7 +28,7 @@ export class LocalFilePersistencePlugin {
|
|
|
28
28
|
return await Pipeline.fromNDJsonFile(filePath).toArray();
|
|
29
29
|
}
|
|
30
30
|
async saveFiles(ops) {
|
|
31
|
-
await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency: 32 });
|
|
31
|
+
await pMap(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 32 });
|
|
32
32
|
}
|
|
33
33
|
async saveFile(table, rows) {
|
|
34
34
|
await fs2.ensureDirAsync(this.cfg.storagePath);
|
package/dist/cnst.js
CHANGED
|
@@ -140,7 +140,7 @@ export class CommonDao {
|
|
|
140
140
|
* Order is not guaranteed, as queries run in parallel.
|
|
141
141
|
*/
|
|
142
142
|
async runUnionQueries(queries, opt) {
|
|
143
|
-
const results = (await pMap(queries, async q => (await this.runQueryExtended(q, opt)).rows)).flat();
|
|
143
|
+
const results = (await pMap(queries, async (q) => (await this.runQueryExtended(q, opt)).rows)).flat();
|
|
144
144
|
return _uniqBy(results, r => r.id);
|
|
145
145
|
}
|
|
146
146
|
async runQueryExtended(q, opt = {}) {
|
|
@@ -274,7 +274,7 @@ export class CommonDao {
|
|
|
274
274
|
* Like patchById, but runs all operations within a Transaction.
|
|
275
275
|
*/
|
|
276
276
|
async patchByIdInTransaction(id, patch, opt) {
|
|
277
|
-
return await this.runInTransaction(async daoTx => {
|
|
277
|
+
return await this.runInTransaction(async (daoTx) => {
|
|
278
278
|
return await this.patchById(id, patch, { ...opt, tx: daoTx.tx });
|
|
279
279
|
});
|
|
280
280
|
}
|
|
@@ -326,7 +326,7 @@ export class CommonDao {
|
|
|
326
326
|
* Like patch, but runs all operations within a Transaction.
|
|
327
327
|
*/
|
|
328
328
|
async patchInTransaction(bm, patch, opt) {
|
|
329
|
-
return await this.runInTransaction(async daoTx => {
|
|
329
|
+
return await this.runInTransaction(async (daoTx) => {
|
|
330
330
|
return await this.patch(bm, patch, { ...opt, tx: daoTx.tx });
|
|
331
331
|
});
|
|
332
332
|
}
|
|
@@ -447,14 +447,14 @@ export class CommonDao {
|
|
|
447
447
|
const { beforeSave } = this.cfg.hooks;
|
|
448
448
|
const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt;
|
|
449
449
|
await p
|
|
450
|
-
.map(async bm => {
|
|
450
|
+
.map(async (bm) => {
|
|
451
451
|
this.assignIdCreatedUpdated(bm, opt);
|
|
452
452
|
const dbm = this.bmToDBM(bm, opt);
|
|
453
453
|
beforeSave?.(dbm);
|
|
454
454
|
return await this.dbmToStorageRow(dbm);
|
|
455
455
|
}, { errorMode })
|
|
456
456
|
.chunk(chunkSize)
|
|
457
|
-
.map(async batch => {
|
|
457
|
+
.map(async (batch) => {
|
|
458
458
|
await this.cfg.db.saveBatch(table, batch, saveOptions);
|
|
459
459
|
return batch;
|
|
460
460
|
}, {
|
|
@@ -501,7 +501,7 @@ export class CommonDao {
|
|
|
501
501
|
.streamQuery(q.select(['id']), opt)
|
|
502
502
|
.mapSync(r => r.id)
|
|
503
503
|
.chunk(chunkSize)
|
|
504
|
-
.map(async ids => {
|
|
504
|
+
.map(async (ids) => {
|
|
505
505
|
await this.cfg.db.deleteByIds(q.table, ids, opt);
|
|
506
506
|
deleted += ids.length;
|
|
507
507
|
}, {
|
|
@@ -613,7 +613,7 @@ export class CommonDao {
|
|
|
613
613
|
async dbmsToStorageRows(dbms) {
|
|
614
614
|
if (!this.cfg.compress?.keys.length)
|
|
615
615
|
return dbms;
|
|
616
|
-
return await pMap(dbms, async dbm => {
|
|
616
|
+
return await pMap(dbms, async (dbm) => {
|
|
617
617
|
const row = { ...dbm };
|
|
618
618
|
await this.compress(row);
|
|
619
619
|
return row;
|
|
@@ -862,7 +862,7 @@ export class CommonDao {
|
|
|
862
862
|
return;
|
|
863
863
|
const { db } = inputs[0].dao.cfg;
|
|
864
864
|
const dbmsByTable = {};
|
|
865
|
-
await pMap(inputs, async input => {
|
|
865
|
+
await pMap(inputs, async (input) => {
|
|
866
866
|
const { dao } = input;
|
|
867
867
|
const { table } = dao.cfg;
|
|
868
868
|
dbmsByTable[table] ||= [];
|
|
@@ -904,7 +904,7 @@ export class CommonDao {
|
|
|
904
904
|
}
|
|
905
905
|
async runInTransaction(fn, opt) {
|
|
906
906
|
let r;
|
|
907
|
-
await this.cfg.db.runInTransaction(async tx => {
|
|
907
|
+
await this.cfg.db.runInTransaction(async (tx) => {
|
|
908
908
|
const daoTx = new CommonDaoTransaction(tx, this.cfg.logger);
|
|
909
909
|
try {
|
|
910
910
|
r = await fn(daoTx);
|
package/dist/db.model.js
CHANGED
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
export
|
|
2
|
-
var DBRelation;
|
|
1
|
+
export var DBRelation;
|
|
3
2
|
(function (DBRelation) {
|
|
4
3
|
DBRelation["ONE_TO_ONE"] = "ONE_TO_ONE";
|
|
5
4
|
DBRelation["ONE_TO_MANY"] = "ONE_TO_MANY";
|
|
6
5
|
})(DBRelation || (DBRelation = {}));
|
|
7
|
-
export
|
|
8
|
-
var DBModelType;
|
|
6
|
+
export var DBModelType;
|
|
9
7
|
(function (DBModelType) {
|
|
10
8
|
DBModelType["DBM"] = "DBM";
|
|
11
9
|
DBModelType["BM"] = "BM";
|
|
@@ -8,8 +8,8 @@ export declare class InMemoryKeyValueDB implements CommonKeyValueDB {
|
|
|
8
8
|
cfg: InMemoryKeyValueDBCfg;
|
|
9
9
|
constructor(cfg?: InMemoryKeyValueDBCfg);
|
|
10
10
|
support: {
|
|
11
|
-
count?: boolean
|
|
12
|
-
increment?: boolean
|
|
11
|
+
count?: boolean;
|
|
12
|
+
increment?: boolean;
|
|
13
13
|
};
|
|
14
14
|
data: StringMap<StringMap<any>>;
|
|
15
15
|
ping(): Promise<void>;
|
package/dist/kv/index.d.ts
CHANGED
package/dist/kv/index.js
CHANGED
|
@@ -23,7 +23,7 @@ export async function dbPipelineBackup(opt) {
|
|
|
23
23
|
tables ||= await db.getTables();
|
|
24
24
|
console.log(`${yellow(tables.length)} ${boldWhite('table(s)')}:\n` + tables.join('\n'));
|
|
25
25
|
const statsPerTable = {};
|
|
26
|
-
await pMap(tables, async table => {
|
|
26
|
+
await pMap(tables, async (table) => {
|
|
27
27
|
let q = DBQuery.create(table).limit(limit);
|
|
28
28
|
const sinceUpdated = opt.sinceUpdatedPerTable?.[table] ?? opt.sinceUpdated;
|
|
29
29
|
if (sinceUpdated) {
|
|
@@ -19,7 +19,7 @@ export async function dbPipelineCopy(opt) {
|
|
|
19
19
|
tables ||= await dbInput.getTables();
|
|
20
20
|
console.log(`${yellow(tables.length)} ${boldWhite('table(s)')}:\n` + tables.join('\n'));
|
|
21
21
|
const statsPerTable = {};
|
|
22
|
-
await pMap(tables, async table => {
|
|
22
|
+
await pMap(tables, async (table) => {
|
|
23
23
|
let q = DBQuery.create(table).limit(limit);
|
|
24
24
|
if (sinceUpdated) {
|
|
25
25
|
q = q.filter('updated', '>=', sinceUpdated);
|
|
@@ -43,7 +43,7 @@ export async function dbPipelineCopy(opt) {
|
|
|
43
43
|
.flattenIfNeeded()
|
|
44
44
|
.tapSync(() => rows++)
|
|
45
45
|
.chunk(chunkSize)
|
|
46
|
-
.forEach(async dbms => {
|
|
46
|
+
.forEach(async (dbms) => {
|
|
47
47
|
await dbOutput.saveBatch(table, dbms, saveOptions);
|
|
48
48
|
});
|
|
49
49
|
const stats = NDJsonStats.create({
|
|
@@ -48,7 +48,7 @@ export async function dbPipelineRestore(opt) {
|
|
|
48
48
|
console.log(`${yellow(tables.length)} ${boldWhite('table(s)')}:\n`, sizeStrByTable);
|
|
49
49
|
// const schemaByTable: Record<string, CommonSchema> = {}
|
|
50
50
|
if (recreateTables) {
|
|
51
|
-
await pMap(tables, async table => {
|
|
51
|
+
await pMap(tables, async (table) => {
|
|
52
52
|
const schemaFilePath = `${inputDirPath}/${table}.schema.json`;
|
|
53
53
|
if (!fs2.pathExists(schemaFilePath)) {
|
|
54
54
|
console.warn(`${schemaFilePath} does not exist!`);
|
|
@@ -58,7 +58,7 @@ export async function dbPipelineRestore(opt) {
|
|
|
58
58
|
await db.createTable(table, schema, { dropIfExists: true });
|
|
59
59
|
});
|
|
60
60
|
}
|
|
61
|
-
await pMap(tables, async table => {
|
|
61
|
+
await pMap(tables, async (table) => {
|
|
62
62
|
const zst = tablesToCompress.has(table);
|
|
63
63
|
const filePath = `${inputDirPath}/${table}.ndjson` + (zst ? '.zst' : '');
|
|
64
64
|
const saveOptions = saveOptionsPerTable[table] || {};
|
|
@@ -82,7 +82,7 @@ export async function dbPipelineRestore(opt) {
|
|
|
82
82
|
})
|
|
83
83
|
.flattenIfNeeded()
|
|
84
84
|
.chunk(chunkSize)
|
|
85
|
-
.forEach(async dbms => {
|
|
85
|
+
.forEach(async (dbms) => {
|
|
86
86
|
await db.saveBatch(table, dbms, saveOptions);
|
|
87
87
|
});
|
|
88
88
|
const stats = NDJsonStats.create({
|
|
@@ -169,7 +169,7 @@ export async function runCommonDBTest(db, quirks = {}) {
|
|
|
169
169
|
const tables = await db.getTables();
|
|
170
170
|
// console.log({ tables })
|
|
171
171
|
if (support.tableSchemas) {
|
|
172
|
-
await pMap(tables, async table => {
|
|
172
|
+
await pMap(tables, async (table) => {
|
|
173
173
|
const schema = await db.getTableSchema(table);
|
|
174
174
|
// console.log(schema)
|
|
175
175
|
expect(schema.$id).toBe(`${table}.schema.json`);
|
|
@@ -227,7 +227,7 @@ export async function runCommonDBTest(db, quirks = {}) {
|
|
|
227
227
|
// save item3 with k1: k1_mod
|
|
228
228
|
// delete item2
|
|
229
229
|
// remaining: item1, item3_with_k1_mod
|
|
230
|
-
await db.runInTransaction(async tx => {
|
|
230
|
+
await db.runInTransaction(async (tx) => {
|
|
231
231
|
await tx.saveBatch(TEST_TABLE, items);
|
|
232
232
|
await tx.saveBatch(TEST_TABLE, [{ ...items[2], k1: 'k1_mod' }]);
|
|
233
233
|
await tx.deleteByIds(TEST_TABLE, [items[1].id]);
|
|
@@ -252,7 +252,7 @@ export async function runCommonDBTest(db, quirks = {}) {
|
|
|
252
252
|
const expected = await prepare();
|
|
253
253
|
let err;
|
|
254
254
|
try {
|
|
255
|
-
await db.runInTransaction(async tx => {
|
|
255
|
+
await db.runInTransaction(async (tx) => {
|
|
256
256
|
await tx.deleteByIds(TEST_TABLE, [items[2].id]);
|
|
257
257
|
// It should fail on id == null
|
|
258
258
|
await tx.saveBatch(TEST_TABLE, [{ ...items[0], k1: 5, id: null }]);
|
|
@@ -242,7 +242,7 @@ export async function runCommonDaoTest(db, quirks = {}) {
|
|
|
242
242
|
await dao.query().deleteByQuery();
|
|
243
243
|
// Test that id, created, updated are created
|
|
244
244
|
const now = localTime.nowUnix();
|
|
245
|
-
await dao.runInTransaction(async tx => {
|
|
245
|
+
await dao.runInTransaction(async (tx) => {
|
|
246
246
|
const row = _omit(item1, ['id', 'created', 'updated']);
|
|
247
247
|
await tx.save(dao, row);
|
|
248
248
|
});
|
|
@@ -251,14 +251,14 @@ export async function runCommonDaoTest(db, quirks = {}) {
|
|
|
251
251
|
expect(loaded[0].id).toBeDefined();
|
|
252
252
|
expect(loaded[0].created).toBeGreaterThanOrEqual(now);
|
|
253
253
|
expect(loaded[0].updated).toBe(loaded[0].created);
|
|
254
|
-
await dao.runInTransaction(async tx => {
|
|
254
|
+
await dao.runInTransaction(async (tx) => {
|
|
255
255
|
await tx.deleteById(dao, loaded[0].id);
|
|
256
256
|
});
|
|
257
257
|
// saveBatch [item1, 2, 3]
|
|
258
258
|
// save item3 with k1: k1_mod
|
|
259
259
|
// delete item2
|
|
260
260
|
// remaining: item1, item3_with_k1_mod
|
|
261
|
-
await dao.runInTransaction(async tx => {
|
|
261
|
+
await dao.runInTransaction(async (tx) => {
|
|
262
262
|
await tx.saveBatch(dao, items);
|
|
263
263
|
await tx.save(dao, { ...items[2], k1: 'k1_mod' });
|
|
264
264
|
await tx.deleteById(dao, items[1].id);
|
|
@@ -303,7 +303,7 @@ export async function runCommonDaoTest(db, quirks = {}) {
|
|
|
303
303
|
const expected = await prepare();
|
|
304
304
|
let err;
|
|
305
305
|
try {
|
|
306
|
-
await dao.runInTransaction(async tx => {
|
|
306
|
+
await dao.runInTransaction(async (tx) => {
|
|
307
307
|
await tx.deleteById(dao, items[2].id);
|
|
308
308
|
await tx.save(dao, { ...items[0], k1: 5 }); // it should fail here
|
|
309
309
|
});
|
|
@@ -40,7 +40,7 @@ export class CommonTimeSeriesDao {
|
|
|
40
40
|
async commitTransaction(ops) {
|
|
41
41
|
if (!ops.length)
|
|
42
42
|
return;
|
|
43
|
-
await this.cfg.db.runInTransaction(async tx => {
|
|
43
|
+
await this.cfg.db.runInTransaction(async (tx) => {
|
|
44
44
|
for (const op of ops) {
|
|
45
45
|
const rows = op.dataPoints.map(([ts, v]) => ({
|
|
46
46
|
id: String(ts), // Convert Number id into String id, as per CommonDB
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
2
|
+
import { JBuilder } from '@naturalcycles/nodejs-lib/ajv';
|
|
1
3
|
import type { CommonDBOptions } from '../db.model.js';
|
|
2
4
|
export declare const commonDBOptionsSchema: () => JBuilder<CommonDBOptions, false>;
|
|
3
5
|
export declare const commonDBSaveOptionsSchema: <ROW extends ObjectWithId>() => any;
|
package/package.json
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@naturalcycles/db-lib",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "10.
|
|
4
|
+
"version": "10.51.0",
|
|
5
5
|
"dependencies": {
|
|
6
6
|
"@naturalcycles/js-lib": "^15",
|
|
7
7
|
"@naturalcycles/nodejs-lib": "^15"
|
|
8
8
|
},
|
|
9
9
|
"devDependencies": {
|
|
10
|
-
"@typescript/native-preview": "7.0.0-dev.
|
|
11
|
-
"@naturalcycles/dev-lib": "
|
|
10
|
+
"@typescript/native-preview": "7.0.0-dev.20260401.1",
|
|
11
|
+
"@naturalcycles/dev-lib": "20.42.0"
|
|
12
12
|
},
|
|
13
13
|
"files": [
|
|
14
14
|
"dist",
|
package/src/kv/index.ts
CHANGED