@naturalcycles/db-lib 10.47.1 → 10.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -143,11 +143,11 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
|
|
|
143
143
|
* const storageRow = await dao.dbmToStorageRow(dbm)
|
|
144
144
|
* await db.saveBatch(table, [storageRow])
|
|
145
145
|
*/
|
|
146
|
-
dbmToStorageRow(dbm: DBM):
|
|
146
|
+
dbmToStorageRow(dbm: DBM): ObjectWithId;
|
|
147
147
|
/**
|
|
148
148
|
* Converts multiple DBMs to storage rows.
|
|
149
149
|
*/
|
|
150
|
-
dbmsToStorageRows(dbms: DBM[]):
|
|
150
|
+
dbmsToStorageRows(dbms: DBM[]): ObjectWithId[];
|
|
151
151
|
/**
|
|
152
152
|
* Converts a storage row back to a DBM, applying decompression if needed.
|
|
153
153
|
*
|
|
@@ -157,12 +157,18 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
|
|
|
157
157
|
* const rows = await db.getByIds(table, ids)
|
|
158
158
|
* const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
|
|
159
159
|
*/
|
|
160
|
-
storageRowToDBM(row: ObjectWithId):
|
|
160
|
+
storageRowToDBM(row: ObjectWithId): DBM;
|
|
161
161
|
/**
|
|
162
162
|
* Converts multiple storage rows to DBMs.
|
|
163
163
|
*/
|
|
164
|
-
|
|
164
|
+
storageRowsToDBM(rows: ObjectWithId[]): DBM[];
|
|
165
|
+
/**
|
|
166
|
+
* Mutates `dbm`.
|
|
167
|
+
*/
|
|
165
168
|
private compress;
|
|
169
|
+
/**
|
|
170
|
+
* Mutates `dbm`.
|
|
171
|
+
*/
|
|
166
172
|
private decompress;
|
|
167
173
|
anyToDBM(dbm: undefined, opt?: CommonDaoOptions): null;
|
|
168
174
|
anyToDBM(dbm?: any, opt?: CommonDaoOptions): DBM;
|
|
@@ -7,7 +7,7 @@ import { _filterUndefinedValues, _objectAssignExact, _omitWithUndefined, _pick,
|
|
|
7
7
|
import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
|
|
8
8
|
import { _objectKeys, _passthroughPredicate, _stringMapEntries, _stringMapValues, _typeCast, } from '@naturalcycles/js-lib/types';
|
|
9
9
|
import { stringId } from '@naturalcycles/nodejs-lib';
|
|
10
|
-
import {
|
|
10
|
+
import { zip2 } from '@naturalcycles/nodejs-lib/zip';
|
|
11
11
|
import { DBLibError } from '../cnst.js';
|
|
12
12
|
import { RunnableDBQuery } from '../query/dbQuery.js';
|
|
13
13
|
import { CommonDaoTransaction } from './commonDaoTransaction.js';
|
|
@@ -105,7 +105,7 @@ export class CommonDao {
|
|
|
105
105
|
return [];
|
|
106
106
|
const table = opt.table || this.cfg.table;
|
|
107
107
|
const rows = await (opt.tx || this.cfg.db).getByIds(table, ids, opt);
|
|
108
|
-
return
|
|
108
|
+
return this.storageRowsToDBM(rows);
|
|
109
109
|
}
|
|
110
110
|
async getBy(by, value, limit = 0, opt) {
|
|
111
111
|
return await this.query().filterEq(by, value).limit(limit).runQuery(opt);
|
|
@@ -148,7 +148,7 @@ export class CommonDao {
|
|
|
148
148
|
q.table = opt.table || q.table;
|
|
149
149
|
const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
|
|
150
150
|
const isPartialQuery = !!q._selectedFieldNames;
|
|
151
|
-
const rows = isPartialQuery ? rawRows :
|
|
151
|
+
const rows = isPartialQuery ? rawRows : this.storageRowsToDBM(rawRows);
|
|
152
152
|
const bms = isPartialQuery ? rows : this.dbmsToBM(rows, opt);
|
|
153
153
|
return {
|
|
154
154
|
rows: bms,
|
|
@@ -164,7 +164,7 @@ export class CommonDao {
|
|
|
164
164
|
q.table = opt.table || q.table;
|
|
165
165
|
const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
|
|
166
166
|
const isPartialQuery = !!q._selectedFieldNames;
|
|
167
|
-
const rows = isPartialQuery ? rawRows :
|
|
167
|
+
const rows = isPartialQuery ? rawRows : this.storageRowsToDBM(rawRows);
|
|
168
168
|
const dbms = isPartialQuery ? rows : this.anyToDBMs(rows, opt);
|
|
169
169
|
return { rows: dbms, ...queryResult };
|
|
170
170
|
}
|
|
@@ -178,10 +178,7 @@ export class CommonDao {
|
|
|
178
178
|
q.table = opt.table || q.table;
|
|
179
179
|
let pipeline = this.cfg.db.streamQuery(q, opt);
|
|
180
180
|
if (this.cfg.compress?.keys.length) {
|
|
181
|
-
pipeline = pipeline.
|
|
182
|
-
// lowered, to reduce the total buffer size a bit for uncompressed objects
|
|
183
|
-
highWaterMark: 16,
|
|
184
|
-
});
|
|
181
|
+
pipeline = pipeline.mapSync(row => this.storageRowToDBM(row));
|
|
185
182
|
}
|
|
186
183
|
const isPartialQuery = !!q._selectedFieldNames;
|
|
187
184
|
if (isPartialQuery)
|
|
@@ -195,10 +192,7 @@ export class CommonDao {
|
|
|
195
192
|
q.table = opt.table || q.table;
|
|
196
193
|
let pipeline = this.cfg.db.streamQuery(q, opt);
|
|
197
194
|
if (this.cfg.compress?.keys.length) {
|
|
198
|
-
pipeline = pipeline.
|
|
199
|
-
// lowered, to reduce the total buffer size a bit for uncompressed objects
|
|
200
|
-
highWaterMark: 16,
|
|
201
|
-
});
|
|
195
|
+
pipeline = pipeline.mapSync(row => this.storageRowToDBM(row));
|
|
202
196
|
}
|
|
203
197
|
const isPartialQuery = !!q._selectedFieldNames;
|
|
204
198
|
if (isPartialQuery)
|
|
@@ -357,7 +351,7 @@ export class CommonDao {
|
|
|
357
351
|
this.cfg.hooks.beforeSave?.(dbm);
|
|
358
352
|
const table = opt.table || this.cfg.table;
|
|
359
353
|
const saveOptions = this.prepareSaveOptions(opt);
|
|
360
|
-
const row =
|
|
354
|
+
const row = this.dbmToStorageRow(dbm);
|
|
361
355
|
await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions);
|
|
362
356
|
if (saveOptions.assignGeneratedIds) {
|
|
363
357
|
bm.id = dbm.id;
|
|
@@ -371,7 +365,7 @@ export class CommonDao {
|
|
|
371
365
|
this.cfg.hooks.beforeSave?.(validDbm);
|
|
372
366
|
const table = opt.table || this.cfg.table;
|
|
373
367
|
const saveOptions = this.prepareSaveOptions(opt);
|
|
374
|
-
const row =
|
|
368
|
+
const row = this.dbmToStorageRow(validDbm);
|
|
375
369
|
await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions);
|
|
376
370
|
if (saveOptions.assignGeneratedIds) {
|
|
377
371
|
dbm.id = validDbm.id;
|
|
@@ -389,7 +383,7 @@ export class CommonDao {
|
|
|
389
383
|
}
|
|
390
384
|
const table = opt.table || this.cfg.table;
|
|
391
385
|
const saveOptions = this.prepareSaveOptions(opt);
|
|
392
|
-
const rows =
|
|
386
|
+
const rows = this.dbmsToStorageRows(dbms);
|
|
393
387
|
await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions);
|
|
394
388
|
if (saveOptions.assignGeneratedIds) {
|
|
395
389
|
dbms.forEach((dbm, i) => (bms[i].id = dbm.id));
|
|
@@ -407,7 +401,7 @@ export class CommonDao {
|
|
|
407
401
|
}
|
|
408
402
|
const table = opt.table || this.cfg.table;
|
|
409
403
|
const saveOptions = this.prepareSaveOptions(opt);
|
|
410
|
-
const rows =
|
|
404
|
+
const rows = this.dbmsToStorageRows(validDbms);
|
|
411
405
|
await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions);
|
|
412
406
|
if (saveOptions.assignGeneratedIds) {
|
|
413
407
|
validDbms.forEach((dbm, i) => (dbms[i].id = dbm.id));
|
|
@@ -457,7 +451,7 @@ export class CommonDao {
|
|
|
457
451
|
this.assignIdCreatedUpdated(bm, opt);
|
|
458
452
|
const dbm = this.bmToDBM(bm, opt);
|
|
459
453
|
beforeSave?.(dbm);
|
|
460
|
-
return
|
|
454
|
+
return this.dbmToStorageRow(dbm);
|
|
461
455
|
}, { errorMode })
|
|
462
456
|
.chunk(chunkSize)
|
|
463
457
|
.map(async batch => {
|
|
@@ -606,20 +600,24 @@ export class CommonDao {
|
|
|
606
600
|
* const storageRow = await dao.dbmToStorageRow(dbm)
|
|
607
601
|
* await db.saveBatch(table, [storageRow])
|
|
608
602
|
*/
|
|
609
|
-
|
|
603
|
+
dbmToStorageRow(dbm) {
|
|
610
604
|
if (!this.cfg.compress?.keys.length)
|
|
611
605
|
return dbm;
|
|
612
606
|
const row = { ...dbm };
|
|
613
|
-
|
|
607
|
+
this.compress(row);
|
|
614
608
|
return row;
|
|
615
609
|
}
|
|
616
610
|
/**
|
|
617
611
|
* Converts multiple DBMs to storage rows.
|
|
618
612
|
*/
|
|
619
|
-
|
|
613
|
+
dbmsToStorageRows(dbms) {
|
|
620
614
|
if (!this.cfg.compress?.keys.length)
|
|
621
615
|
return dbms;
|
|
622
|
-
return
|
|
616
|
+
return dbms.map(dbm => {
|
|
617
|
+
const row = { ...dbm };
|
|
618
|
+
this.compress(row);
|
|
619
|
+
return row;
|
|
620
|
+
});
|
|
623
621
|
}
|
|
624
622
|
/**
|
|
625
623
|
* Converts a storage row back to a DBM, applying decompression if needed.
|
|
@@ -630,43 +628,46 @@ export class CommonDao {
|
|
|
630
628
|
* const rows = await db.getByIds(table, ids)
|
|
631
629
|
* const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
|
|
632
630
|
*/
|
|
633
|
-
|
|
631
|
+
storageRowToDBM(row) {
|
|
634
632
|
if (!this.cfg.compress?.keys.length)
|
|
635
633
|
return row;
|
|
636
634
|
const dbm = { ...row };
|
|
637
|
-
|
|
635
|
+
this.decompress(dbm);
|
|
638
636
|
return dbm;
|
|
639
637
|
}
|
|
640
638
|
/**
|
|
641
639
|
* Converts multiple storage rows to DBMs.
|
|
642
640
|
*/
|
|
643
|
-
|
|
641
|
+
storageRowsToDBM(rows) {
|
|
644
642
|
if (!this.cfg.compress?.keys.length)
|
|
645
643
|
return rows;
|
|
646
|
-
return
|
|
644
|
+
return rows.map(row => {
|
|
645
|
+
const dbm = { ...row };
|
|
646
|
+
this.decompress(dbm);
|
|
647
|
+
return dbm;
|
|
648
|
+
});
|
|
647
649
|
}
|
|
648
650
|
/**
|
|
649
651
|
* Mutates `dbm`.
|
|
650
652
|
*/
|
|
651
|
-
|
|
653
|
+
compress(dbm) {
|
|
652
654
|
if (!this.cfg.compress?.keys.length)
|
|
653
655
|
return; // No compression requested
|
|
654
656
|
const { keys, level = 1 } = this.cfg.compress;
|
|
655
657
|
const properties = _pick(dbm, keys);
|
|
656
658
|
const bufferString = JSON.stringify(properties);
|
|
657
|
-
const __compressed =
|
|
659
|
+
const __compressed = zip2.zstdCompressSync(bufferString, level);
|
|
658
660
|
_omitWithUndefined(dbm, _objectKeys(properties), { mutate: true });
|
|
659
661
|
Object.assign(dbm, { __compressed });
|
|
660
662
|
}
|
|
661
663
|
/**
|
|
662
664
|
* Mutates `dbm`.
|
|
663
665
|
*/
|
|
664
|
-
|
|
666
|
+
decompress(dbm) {
|
|
665
667
|
_typeCast(dbm);
|
|
666
668
|
if (!Buffer.isBuffer(dbm.__compressed))
|
|
667
669
|
return; // No compressed data
|
|
668
|
-
|
|
669
|
-
const bufferString = await decompressZstdOrInflateToString(dbm.__compressed);
|
|
670
|
+
const bufferString = zip2.zstdDecompressToStringSync(dbm.__compressed);
|
|
670
671
|
const properties = JSON.parse(bufferString);
|
|
671
672
|
dbm.__compressed = undefined;
|
|
672
673
|
Object.assign(dbm, properties);
|
|
@@ -769,7 +770,7 @@ export class CommonDao {
|
|
|
769
770
|
// todo: support tx
|
|
770
771
|
const dbmsByTable = await db.multiGet(idsByTable, opt);
|
|
771
772
|
const dbmByTableById = CommonDao.multiGetMapByTableById(dbmsByTable);
|
|
772
|
-
return
|
|
773
|
+
return CommonDao.prepareMultiGetOutput(inputMap, dbmByTableById, opt);
|
|
773
774
|
}
|
|
774
775
|
static prepareMultiGetIds(inputMap) {
|
|
775
776
|
const idSetByTable = {};
|
|
@@ -806,17 +807,17 @@ export class CommonDao {
|
|
|
806
807
|
}
|
|
807
808
|
return dbmByTableById;
|
|
808
809
|
}
|
|
809
|
-
static
|
|
810
|
+
static prepareMultiGetOutput(inputMap, dbmByTableById, opt = {}) {
|
|
810
811
|
const bmsByProp = {};
|
|
811
812
|
// Loop over input props again, to produce the output of the same shape as requested
|
|
812
|
-
|
|
813
|
+
for (const [prop, input] of _stringMapEntries(inputMap)) {
|
|
813
814
|
const { dao } = input;
|
|
814
815
|
const { table } = dao.cfg;
|
|
815
816
|
if ('id' in input) {
|
|
816
817
|
// Singular
|
|
817
818
|
const row = dbmByTableById[table][input.id];
|
|
818
819
|
// Decompress before converting to BM
|
|
819
|
-
const dbm = row ?
|
|
820
|
+
const dbm = row ? dao.storageRowToDBM(row) : undefined;
|
|
820
821
|
bmsByProp[prop] = dao.dbmToBM(dbm, opt) || null;
|
|
821
822
|
}
|
|
822
823
|
else {
|
|
@@ -825,10 +826,10 @@ export class CommonDao {
|
|
|
825
826
|
// Without filtering - every prop will get ALL rows from that table.
|
|
826
827
|
const rows = input.ids.map(id => dbmByTableById[table][id]).filter(_isTruthy);
|
|
827
828
|
// Decompress before converting to BM
|
|
828
|
-
const dbms =
|
|
829
|
+
const dbms = dao.storageRowsToDBM(rows);
|
|
829
830
|
bmsByProp[prop] = dao.dbmsToBM(dbms, opt);
|
|
830
831
|
}
|
|
831
|
-
}
|
|
832
|
+
}
|
|
832
833
|
return bmsByProp;
|
|
833
834
|
}
|
|
834
835
|
/**
|
|
@@ -859,7 +860,7 @@ export class CommonDao {
|
|
|
859
860
|
return;
|
|
860
861
|
const { db } = inputs[0].dao.cfg;
|
|
861
862
|
const dbmsByTable = {};
|
|
862
|
-
|
|
863
|
+
for (const input of inputs) {
|
|
863
864
|
const { dao } = input;
|
|
864
865
|
const { table } = dao.cfg;
|
|
865
866
|
dbmsByTable[table] ||= [];
|
|
@@ -879,7 +880,7 @@ export class CommonDao {
|
|
|
879
880
|
dao.assignIdCreatedUpdated(row, opt);
|
|
880
881
|
const dbm = dao.bmToDBM(row, opt);
|
|
881
882
|
dao.cfg.hooks.beforeSave?.(dbm);
|
|
882
|
-
const storageRow =
|
|
883
|
+
const storageRow = dao.dbmToStorageRow(dbm);
|
|
883
884
|
dbmsByTable[table].push(storageRow);
|
|
884
885
|
}
|
|
885
886
|
else {
|
|
@@ -889,10 +890,10 @@ export class CommonDao {
|
|
|
889
890
|
if (dao.cfg.hooks.beforeSave) {
|
|
890
891
|
dbms.forEach(dbm => dao.cfg.hooks.beforeSave(dbm));
|
|
891
892
|
}
|
|
892
|
-
const storageRows =
|
|
893
|
+
const storageRows = dao.dbmsToStorageRows(dbms);
|
|
893
894
|
dbmsByTable[table].push(...storageRows);
|
|
894
895
|
}
|
|
895
|
-
}
|
|
896
|
+
}
|
|
896
897
|
await db.multiSave(dbmsByTable);
|
|
897
898
|
}
|
|
898
899
|
async createTransaction(opt) {
|
|
@@ -28,8 +28,8 @@ export interface CommonKeyValueDaoCfg<V> {
|
|
|
28
28
|
}
|
|
29
29
|
export type CommonKeyValueDaoSaveOptions = CommonKeyValueDBSaveBatchOptions;
|
|
30
30
|
export interface CommonKeyValueDaoTransformer<V> {
|
|
31
|
-
valueToBuffer: (v: V) =>
|
|
32
|
-
bufferToValue: (buf: Buffer) =>
|
|
31
|
+
valueToBuffer: (v: V) => Buffer;
|
|
32
|
+
bufferToValue: (buf: Buffer) => V;
|
|
33
33
|
}
|
|
34
34
|
/**
|
|
35
35
|
* @deprecated use zstd instead, gzip is obsolete
|
|
@@ -1,20 +1,19 @@
|
|
|
1
1
|
import { AppError } from '@naturalcycles/js-lib/error/error.util.js';
|
|
2
|
-
import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
|
|
3
2
|
import { SKIP } from '@naturalcycles/js-lib/types';
|
|
4
|
-
import {
|
|
3
|
+
import { zip2 } from '@naturalcycles/nodejs-lib/zip';
|
|
5
4
|
/**
|
|
6
5
|
* @deprecated use zstd instead, gzip is obsolete
|
|
7
6
|
*/
|
|
8
7
|
export function commonKeyValueDaoDeflatedJsonTransformer() {
|
|
9
8
|
return {
|
|
10
|
-
valueToBuffer:
|
|
11
|
-
bufferToValue:
|
|
9
|
+
valueToBuffer: v => zip2.deflateSync(JSON.stringify(v)),
|
|
10
|
+
bufferToValue: buf => JSON.parse(zip2.inflateToStringSync(buf)),
|
|
12
11
|
};
|
|
13
12
|
}
|
|
14
13
|
export function commonKeyValueDaoZstdJsonTransformer(level) {
|
|
15
14
|
return {
|
|
16
|
-
valueToBuffer:
|
|
17
|
-
bufferToValue:
|
|
15
|
+
valueToBuffer: v => zip2.zstdCompressSync(JSON.stringify(v), level),
|
|
16
|
+
bufferToValue: buf => JSON.parse(zip2.zstdDecompressToStringSync(buf)),
|
|
18
17
|
};
|
|
19
18
|
}
|
|
20
19
|
/**
|
|
@@ -23,8 +22,8 @@ export function commonKeyValueDaoZstdJsonTransformer(level) {
|
|
|
23
22
|
*/
|
|
24
23
|
export function commonKeyValueDaoCompressedTransformer() {
|
|
25
24
|
return {
|
|
26
|
-
valueToBuffer:
|
|
27
|
-
bufferToValue:
|
|
25
|
+
valueToBuffer: v => zip2.zstdCompressSync(JSON.stringify(v)),
|
|
26
|
+
bufferToValue: buf => JSON.parse(zip2.decompressZstdOrInflateToStringSync(buf)),
|
|
28
27
|
};
|
|
29
28
|
}
|
|
30
29
|
// todo: logging
|
|
@@ -81,10 +80,7 @@ export class CommonKeyValueDao {
|
|
|
81
80
|
const entries = await this.cfg.db.getByIds(this.cfg.table, ids);
|
|
82
81
|
if (!this.cfg.transformer)
|
|
83
82
|
return entries;
|
|
84
|
-
return
|
|
85
|
-
id,
|
|
86
|
-
await this.cfg.transformer.bufferToValue(raw),
|
|
87
|
-
]);
|
|
83
|
+
return entries.map(([id, raw]) => [id, this.cfg.transformer.bufferToValue(raw)]);
|
|
88
84
|
}
|
|
89
85
|
async getByIdsAsBuffer(ids) {
|
|
90
86
|
return await this.cfg.db.getByIds(this.cfg.table, ids);
|
|
@@ -99,7 +95,7 @@ export class CommonKeyValueDao {
|
|
|
99
95
|
rawEntries = entries;
|
|
100
96
|
}
|
|
101
97
|
else {
|
|
102
|
-
rawEntries =
|
|
98
|
+
rawEntries = entries.map(([id, v]) => [id, transformer.valueToBuffer(v)]);
|
|
103
99
|
}
|
|
104
100
|
await this.cfg.db.saveBatch(this.cfg.table, rawEntries, opt);
|
|
105
101
|
}
|
|
@@ -117,30 +113,30 @@ export class CommonKeyValueDao {
|
|
|
117
113
|
if (!transformer) {
|
|
118
114
|
return this.cfg.db.streamValues(this.cfg.table, limit);
|
|
119
115
|
}
|
|
120
|
-
return this.cfg.db.streamValues(this.cfg.table, limit).
|
|
116
|
+
return this.cfg.db.streamValues(this.cfg.table, limit).mapSync(buf => {
|
|
121
117
|
try {
|
|
122
|
-
return
|
|
118
|
+
return transformer.bufferToValue(buf);
|
|
123
119
|
}
|
|
124
120
|
catch (err) {
|
|
125
121
|
this.cfg.logger.error(err);
|
|
126
122
|
return SKIP;
|
|
127
123
|
}
|
|
128
|
-
}
|
|
124
|
+
});
|
|
129
125
|
}
|
|
130
126
|
streamEntries(limit) {
|
|
131
127
|
const { transformer } = this.cfg;
|
|
132
128
|
if (!transformer) {
|
|
133
129
|
return this.cfg.db.streamEntries(this.cfg.table, limit);
|
|
134
130
|
}
|
|
135
|
-
return this.cfg.db.streamEntries(this.cfg.table, limit).
|
|
131
|
+
return this.cfg.db.streamEntries(this.cfg.table, limit).mapSync(([id, buf]) => {
|
|
136
132
|
try {
|
|
137
|
-
return [id,
|
|
133
|
+
return [id, transformer.bufferToValue(buf)];
|
|
138
134
|
}
|
|
139
135
|
catch (err) {
|
|
140
136
|
this.cfg.logger.error(err);
|
|
141
137
|
return SKIP;
|
|
142
138
|
}
|
|
143
|
-
}
|
|
139
|
+
});
|
|
144
140
|
}
|
|
145
141
|
async getAllKeys(limit) {
|
|
146
142
|
return await this.streamIds(limit).toArray();
|
package/package.json
CHANGED
|
@@ -27,7 +27,7 @@ import type {
|
|
|
27
27
|
import { stringId } from '@naturalcycles/nodejs-lib'
|
|
28
28
|
import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv'
|
|
29
29
|
import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
|
|
30
|
-
import {
|
|
30
|
+
import { zip2 } from '@naturalcycles/nodejs-lib/zip'
|
|
31
31
|
import { DBLibError } from '../cnst.js'
|
|
32
32
|
import type {
|
|
33
33
|
CommonDBSaveOptions,
|
|
@@ -155,7 +155,7 @@ export class CommonDao<
|
|
|
155
155
|
if (!ids.length) return []
|
|
156
156
|
const table = opt.table || this.cfg.table
|
|
157
157
|
const rows = await (opt.tx || this.cfg.db).getByIds<DBM>(table, ids, opt)
|
|
158
|
-
return
|
|
158
|
+
return this.storageRowsToDBM(rows)
|
|
159
159
|
}
|
|
160
160
|
|
|
161
161
|
async getBy(by: keyof DBM, value: any, limit = 0, opt?: CommonDaoReadOptions): Promise<BM[]> {
|
|
@@ -216,7 +216,7 @@ export class CommonDao<
|
|
|
216
216
|
q.table = opt.table || q.table
|
|
217
217
|
const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
|
|
218
218
|
const isPartialQuery = !!q._selectedFieldNames
|
|
219
|
-
const rows = isPartialQuery ? rawRows :
|
|
219
|
+
const rows = isPartialQuery ? rawRows : this.storageRowsToDBM(rawRows)
|
|
220
220
|
const bms = isPartialQuery ? (rows as any[]) : this.dbmsToBM(rows, opt)
|
|
221
221
|
return {
|
|
222
222
|
rows: bms,
|
|
@@ -237,7 +237,7 @@ export class CommonDao<
|
|
|
237
237
|
q.table = opt.table || q.table
|
|
238
238
|
const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
|
|
239
239
|
const isPartialQuery = !!q._selectedFieldNames
|
|
240
|
-
const rows = isPartialQuery ? rawRows :
|
|
240
|
+
const rows = isPartialQuery ? rawRows : this.storageRowsToDBM(rawRows)
|
|
241
241
|
const dbms = isPartialQuery ? rows : this.anyToDBMs(rows, opt)
|
|
242
242
|
return { rows: dbms, ...queryResult }
|
|
243
243
|
}
|
|
@@ -254,10 +254,7 @@ export class CommonDao<
|
|
|
254
254
|
let pipeline = this.cfg.db.streamQuery<DBM>(q, opt)
|
|
255
255
|
|
|
256
256
|
if (this.cfg.compress?.keys.length) {
|
|
257
|
-
pipeline = pipeline.
|
|
258
|
-
// lowered, to reduce the total buffer size a bit for uncompressed objects
|
|
259
|
-
highWaterMark: 16,
|
|
260
|
-
})
|
|
257
|
+
pipeline = pipeline.mapSync(row => this.storageRowToDBM(row))
|
|
261
258
|
}
|
|
262
259
|
|
|
263
260
|
const isPartialQuery = !!q._selectedFieldNames
|
|
@@ -275,10 +272,7 @@ export class CommonDao<
|
|
|
275
272
|
let pipeline = this.cfg.db.streamQuery<DBM>(q, opt)
|
|
276
273
|
|
|
277
274
|
if (this.cfg.compress?.keys.length) {
|
|
278
|
-
pipeline = pipeline.
|
|
279
|
-
// lowered, to reduce the total buffer size a bit for uncompressed objects
|
|
280
|
-
highWaterMark: 16,
|
|
281
|
-
})
|
|
275
|
+
pipeline = pipeline.mapSync(row => this.storageRowToDBM(row))
|
|
282
276
|
}
|
|
283
277
|
|
|
284
278
|
const isPartialQuery = !!q._selectedFieldNames
|
|
@@ -477,7 +471,7 @@ export class CommonDao<
|
|
|
477
471
|
const table = opt.table || this.cfg.table
|
|
478
472
|
const saveOptions = this.prepareSaveOptions(opt)
|
|
479
473
|
|
|
480
|
-
const row =
|
|
474
|
+
const row = this.dbmToStorageRow(dbm)
|
|
481
475
|
await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions)
|
|
482
476
|
|
|
483
477
|
if (saveOptions.assignGeneratedIds) {
|
|
@@ -495,7 +489,7 @@ export class CommonDao<
|
|
|
495
489
|
const table = opt.table || this.cfg.table
|
|
496
490
|
const saveOptions = this.prepareSaveOptions(opt)
|
|
497
491
|
|
|
498
|
-
const row =
|
|
492
|
+
const row = this.dbmToStorageRow(validDbm)
|
|
499
493
|
await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions)
|
|
500
494
|
|
|
501
495
|
if (saveOptions.assignGeneratedIds) {
|
|
@@ -516,7 +510,7 @@ export class CommonDao<
|
|
|
516
510
|
const table = opt.table || this.cfg.table
|
|
517
511
|
const saveOptions = this.prepareSaveOptions(opt)
|
|
518
512
|
|
|
519
|
-
const rows =
|
|
513
|
+
const rows = this.dbmsToStorageRows(dbms)
|
|
520
514
|
await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions)
|
|
521
515
|
|
|
522
516
|
if (saveOptions.assignGeneratedIds) {
|
|
@@ -540,7 +534,7 @@ export class CommonDao<
|
|
|
540
534
|
const table = opt.table || this.cfg.table
|
|
541
535
|
const saveOptions = this.prepareSaveOptions(opt)
|
|
542
536
|
|
|
543
|
-
const rows =
|
|
537
|
+
const rows = this.dbmsToStorageRows(validDbms)
|
|
544
538
|
await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions)
|
|
545
539
|
|
|
546
540
|
if (saveOptions.assignGeneratedIds) {
|
|
@@ -608,7 +602,7 @@ export class CommonDao<
|
|
|
608
602
|
this.assignIdCreatedUpdated(bm, opt)
|
|
609
603
|
const dbm = this.bmToDBM(bm, opt)
|
|
610
604
|
beforeSave?.(dbm)
|
|
611
|
-
return
|
|
605
|
+
return this.dbmToStorageRow(dbm)
|
|
612
606
|
},
|
|
613
607
|
{ errorMode },
|
|
614
608
|
)
|
|
@@ -802,19 +796,23 @@ export class CommonDao<
|
|
|
802
796
|
* const storageRow = await dao.dbmToStorageRow(dbm)
|
|
803
797
|
* await db.saveBatch(table, [storageRow])
|
|
804
798
|
*/
|
|
805
|
-
|
|
799
|
+
dbmToStorageRow(dbm: DBM): ObjectWithId {
|
|
806
800
|
if (!this.cfg.compress?.keys.length) return dbm
|
|
807
801
|
const row = { ...dbm }
|
|
808
|
-
|
|
802
|
+
this.compress(row)
|
|
809
803
|
return row
|
|
810
804
|
}
|
|
811
805
|
|
|
812
806
|
/**
|
|
813
807
|
* Converts multiple DBMs to storage rows.
|
|
814
808
|
*/
|
|
815
|
-
|
|
809
|
+
dbmsToStorageRows(dbms: DBM[]): ObjectWithId[] {
|
|
816
810
|
if (!this.cfg.compress?.keys.length) return dbms
|
|
817
|
-
return
|
|
811
|
+
return dbms.map(dbm => {
|
|
812
|
+
const row = { ...dbm }
|
|
813
|
+
this.compress(row)
|
|
814
|
+
return row
|
|
815
|
+
})
|
|
818
816
|
}
|
|
819
817
|
|
|
820
818
|
/**
|
|
@@ -826,31 +824,35 @@ export class CommonDao<
|
|
|
826
824
|
* const rows = await db.getByIds(table, ids)
|
|
827
825
|
* const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
|
|
828
826
|
*/
|
|
829
|
-
|
|
827
|
+
storageRowToDBM(row: ObjectWithId): DBM {
|
|
830
828
|
if (!this.cfg.compress?.keys.length) return row as DBM
|
|
831
829
|
const dbm = { ...(row as DBM) }
|
|
832
|
-
|
|
830
|
+
this.decompress(dbm)
|
|
833
831
|
return dbm
|
|
834
832
|
}
|
|
835
833
|
|
|
836
834
|
/**
|
|
837
835
|
* Converts multiple storage rows to DBMs.
|
|
838
836
|
*/
|
|
839
|
-
|
|
837
|
+
storageRowsToDBM(rows: ObjectWithId[]): DBM[] {
|
|
840
838
|
if (!this.cfg.compress?.keys.length) return rows as DBM[]
|
|
841
|
-
return
|
|
839
|
+
return rows.map(row => {
|
|
840
|
+
const dbm = { ...(row as DBM) }
|
|
841
|
+
this.decompress(dbm)
|
|
842
|
+
return dbm
|
|
843
|
+
})
|
|
842
844
|
}
|
|
843
845
|
|
|
844
846
|
/**
|
|
845
847
|
* Mutates `dbm`.
|
|
846
848
|
*/
|
|
847
|
-
private
|
|
849
|
+
private compress(dbm: DBM): void {
|
|
848
850
|
if (!this.cfg.compress?.keys.length) return // No compression requested
|
|
849
851
|
|
|
850
852
|
const { keys, level = 1 } = this.cfg.compress
|
|
851
853
|
const properties = _pick(dbm, keys)
|
|
852
854
|
const bufferString = JSON.stringify(properties)
|
|
853
|
-
const __compressed =
|
|
855
|
+
const __compressed = zip2.zstdCompressSync(bufferString, level)
|
|
854
856
|
_omitWithUndefined(dbm as any, _objectKeys(properties), { mutate: true })
|
|
855
857
|
Object.assign(dbm, { __compressed })
|
|
856
858
|
}
|
|
@@ -858,12 +860,11 @@ export class CommonDao<
|
|
|
858
860
|
/**
|
|
859
861
|
* Mutates `dbm`.
|
|
860
862
|
*/
|
|
861
|
-
private
|
|
863
|
+
private decompress(dbm: DBM): void {
|
|
862
864
|
_typeCast<Compressed<DBM>>(dbm)
|
|
863
865
|
if (!Buffer.isBuffer(dbm.__compressed)) return // No compressed data
|
|
864
866
|
|
|
865
|
-
|
|
866
|
-
const bufferString = await decompressZstdOrInflateToString(dbm.__compressed)
|
|
867
|
+
const bufferString = zip2.zstdDecompressToStringSync(dbm.__compressed)
|
|
867
868
|
const properties = JSON.parse(bufferString)
|
|
868
869
|
dbm.__compressed = undefined
|
|
869
870
|
Object.assign(dbm, properties)
|
|
@@ -1001,7 +1002,7 @@ export class CommonDao<
|
|
|
1001
1002
|
|
|
1002
1003
|
const dbmByTableById = CommonDao.multiGetMapByTableById(dbmsByTable)
|
|
1003
1004
|
|
|
1004
|
-
return
|
|
1005
|
+
return CommonDao.prepareMultiGetOutput(inputMap, dbmByTableById, opt) as any
|
|
1005
1006
|
}
|
|
1006
1007
|
|
|
1007
1008
|
private static prepareMultiGetIds(
|
|
@@ -1047,22 +1048,22 @@ export class CommonDao<
|
|
|
1047
1048
|
return dbmByTableById
|
|
1048
1049
|
}
|
|
1049
1050
|
|
|
1050
|
-
private static
|
|
1051
|
+
private static prepareMultiGetOutput(
|
|
1051
1052
|
inputMap: StringMap<DaoWithIds<AnyDao> | DaoWithId<AnyDao>>,
|
|
1052
1053
|
dbmByTableById: StringMap<StringMap<ObjectWithId>>,
|
|
1053
1054
|
opt: CommonDaoReadOptions = {},
|
|
1054
|
-
):
|
|
1055
|
+
): StringMap<unknown> {
|
|
1055
1056
|
const bmsByProp: StringMap<unknown> = {}
|
|
1056
1057
|
|
|
1057
1058
|
// Loop over input props again, to produce the output of the same shape as requested
|
|
1058
|
-
|
|
1059
|
+
for (const [prop, input] of _stringMapEntries(inputMap)) {
|
|
1059
1060
|
const { dao } = input
|
|
1060
1061
|
const { table } = dao.cfg
|
|
1061
1062
|
if ('id' in input) {
|
|
1062
1063
|
// Singular
|
|
1063
1064
|
const row = dbmByTableById[table]![input.id]
|
|
1064
1065
|
// Decompress before converting to BM
|
|
1065
|
-
const dbm = row ?
|
|
1066
|
+
const dbm = row ? dao.storageRowToDBM(row) : undefined
|
|
1066
1067
|
bmsByProp[prop] = dao.dbmToBM(dbm, opt) || null
|
|
1067
1068
|
} else {
|
|
1068
1069
|
// Plural
|
|
@@ -1070,10 +1071,10 @@ export class CommonDao<
|
|
|
1070
1071
|
// Without filtering - every prop will get ALL rows from that table.
|
|
1071
1072
|
const rows = input.ids.map(id => dbmByTableById[table]![id]).filter(_isTruthy)
|
|
1072
1073
|
// Decompress before converting to BM
|
|
1073
|
-
const dbms =
|
|
1074
|
+
const dbms = dao.storageRowsToDBM(rows)
|
|
1074
1075
|
bmsByProp[prop] = dao.dbmsToBM(dbms, opt)
|
|
1075
1076
|
}
|
|
1076
|
-
}
|
|
1077
|
+
}
|
|
1077
1078
|
|
|
1078
1079
|
return bmsByProp as any
|
|
1079
1080
|
}
|
|
@@ -1112,7 +1113,7 @@ export class CommonDao<
|
|
|
1112
1113
|
if (!inputs.length) return
|
|
1113
1114
|
const { db } = inputs[0]!.dao.cfg
|
|
1114
1115
|
const dbmsByTable: StringMap<any[]> = {}
|
|
1115
|
-
|
|
1116
|
+
for (const input of inputs) {
|
|
1116
1117
|
const { dao } = input
|
|
1117
1118
|
const { table } = dao.cfg
|
|
1118
1119
|
dbmsByTable[table] ||= []
|
|
@@ -1135,7 +1136,7 @@ export class CommonDao<
|
|
|
1135
1136
|
dao.assignIdCreatedUpdated(row, opt)
|
|
1136
1137
|
const dbm = dao.bmToDBM(row, opt)
|
|
1137
1138
|
dao.cfg.hooks!.beforeSave?.(dbm)
|
|
1138
|
-
const storageRow =
|
|
1139
|
+
const storageRow = dao.dbmToStorageRow(dbm)
|
|
1139
1140
|
dbmsByTable[table].push(storageRow)
|
|
1140
1141
|
} else {
|
|
1141
1142
|
// Plural
|
|
@@ -1144,10 +1145,10 @@ export class CommonDao<
|
|
|
1144
1145
|
if (dao.cfg.hooks!.beforeSave) {
|
|
1145
1146
|
dbms.forEach(dbm => dao.cfg.hooks!.beforeSave!(dbm))
|
|
1146
1147
|
}
|
|
1147
|
-
const storageRows =
|
|
1148
|
+
const storageRows = dao.dbmsToStorageRows(dbms)
|
|
1148
1149
|
dbmsByTable[table].push(...storageRows)
|
|
1149
1150
|
}
|
|
1150
|
-
}
|
|
1151
|
+
}
|
|
1151
1152
|
|
|
1152
1153
|
await db.multiSave(dbmsByTable)
|
|
1153
1154
|
}
|
|
@@ -1,16 +1,9 @@
|
|
|
1
1
|
import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
|
|
2
2
|
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
3
|
-
import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
|
|
4
3
|
import { SKIP } from '@naturalcycles/js-lib/types'
|
|
5
4
|
import type { Integer, KeyValueTuple } from '@naturalcycles/js-lib/types'
|
|
6
5
|
import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
|
|
7
|
-
import {
|
|
8
|
-
decompressZstdOrInflateToString,
|
|
9
|
-
deflateString,
|
|
10
|
-
inflateToString,
|
|
11
|
-
zstdCompress,
|
|
12
|
-
zstdDecompressToString,
|
|
13
|
-
} from '@naturalcycles/nodejs-lib/zip'
|
|
6
|
+
import { zip2 } from '@naturalcycles/nodejs-lib/zip'
|
|
14
7
|
import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js'
|
|
15
8
|
import type { CommonDBCreateOptions } from '../db.model.js'
|
|
16
9
|
import type {
|
|
@@ -52,8 +45,8 @@ export interface CommonKeyValueDaoCfg<V> {
|
|
|
52
45
|
export type CommonKeyValueDaoSaveOptions = CommonKeyValueDBSaveBatchOptions
|
|
53
46
|
|
|
54
47
|
export interface CommonKeyValueDaoTransformer<V> {
|
|
55
|
-
valueToBuffer: (v: V) =>
|
|
56
|
-
bufferToValue: (buf: Buffer) =>
|
|
48
|
+
valueToBuffer: (v: V) => Buffer
|
|
49
|
+
bufferToValue: (buf: Buffer) => V
|
|
57
50
|
}
|
|
58
51
|
|
|
59
52
|
/**
|
|
@@ -63,8 +56,8 @@ export function commonKeyValueDaoDeflatedJsonTransformer<
|
|
|
63
56
|
T = any,
|
|
64
57
|
>(): CommonKeyValueDaoTransformer<T> {
|
|
65
58
|
return {
|
|
66
|
-
valueToBuffer:
|
|
67
|
-
bufferToValue:
|
|
59
|
+
valueToBuffer: v => zip2.deflateSync(JSON.stringify(v)),
|
|
60
|
+
bufferToValue: buf => JSON.parse(zip2.inflateToStringSync(buf)),
|
|
68
61
|
}
|
|
69
62
|
}
|
|
70
63
|
|
|
@@ -72,8 +65,8 @@ export function commonKeyValueDaoZstdJsonTransformer<T = any>(
|
|
|
72
65
|
level: Integer | undefined, // defaults to 3
|
|
73
66
|
): CommonKeyValueDaoTransformer<T> {
|
|
74
67
|
return {
|
|
75
|
-
valueToBuffer:
|
|
76
|
-
bufferToValue:
|
|
68
|
+
valueToBuffer: v => zip2.zstdCompressSync(JSON.stringify(v), level),
|
|
69
|
+
bufferToValue: buf => JSON.parse(zip2.zstdDecompressToStringSync(buf)),
|
|
77
70
|
}
|
|
78
71
|
}
|
|
79
72
|
|
|
@@ -83,8 +76,8 @@ export function commonKeyValueDaoZstdJsonTransformer<T = any>(
|
|
|
83
76
|
*/
|
|
84
77
|
export function commonKeyValueDaoCompressedTransformer<T = any>(): CommonKeyValueDaoTransformer<T> {
|
|
85
78
|
return {
|
|
86
|
-
valueToBuffer:
|
|
87
|
-
bufferToValue:
|
|
79
|
+
valueToBuffer: v => zip2.zstdCompressSync(JSON.stringify(v)),
|
|
80
|
+
bufferToValue: buf => JSON.parse(zip2.decompressZstdOrInflateToStringSync(buf)),
|
|
88
81
|
}
|
|
89
82
|
}
|
|
90
83
|
|
|
@@ -155,10 +148,7 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
|
|
|
155
148
|
const entries = await this.cfg.db.getByIds(this.cfg.table, ids)
|
|
156
149
|
if (!this.cfg.transformer) return entries as any
|
|
157
150
|
|
|
158
|
-
return
|
|
159
|
-
id,
|
|
160
|
-
await this.cfg.transformer!.bufferToValue(raw),
|
|
161
|
-
])
|
|
151
|
+
return entries.map(([id, raw]) => [id, this.cfg.transformer!.bufferToValue(raw)])
|
|
162
152
|
}
|
|
163
153
|
|
|
164
154
|
async getByIdsAsBuffer(ids: K[]): Promise<KeyValueDBTuple[]> {
|
|
@@ -179,7 +169,7 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
|
|
|
179
169
|
if (!transformer) {
|
|
180
170
|
rawEntries = entries as any
|
|
181
171
|
} else {
|
|
182
|
-
rawEntries =
|
|
172
|
+
rawEntries = entries.map(([id, v]) => [id, transformer.valueToBuffer(v)])
|
|
183
173
|
}
|
|
184
174
|
|
|
185
175
|
await this.cfg.db.saveBatch(this.cfg.table, rawEntries, opt)
|
|
@@ -204,17 +194,14 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
|
|
|
204
194
|
return this.cfg.db.streamValues(this.cfg.table, limit) as Pipeline<V>
|
|
205
195
|
}
|
|
206
196
|
|
|
207
|
-
return this.cfg.db.streamValues(this.cfg.table, limit).
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
},
|
|
216
|
-
{ concurrency: 32 },
|
|
217
|
-
)
|
|
197
|
+
return this.cfg.db.streamValues(this.cfg.table, limit).mapSync(buf => {
|
|
198
|
+
try {
|
|
199
|
+
return transformer.bufferToValue(buf)
|
|
200
|
+
} catch (err) {
|
|
201
|
+
this.cfg.logger.error(err)
|
|
202
|
+
return SKIP
|
|
203
|
+
}
|
|
204
|
+
})
|
|
218
205
|
}
|
|
219
206
|
|
|
220
207
|
streamEntries(limit?: number): Pipeline<KeyValueTuple<K, V>> {
|
|
@@ -224,17 +211,14 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
|
|
|
224
211
|
return this.cfg.db.streamEntries(this.cfg.table, limit) as Pipeline<KeyValueTuple<K, V>>
|
|
225
212
|
}
|
|
226
213
|
|
|
227
|
-
return this.cfg.db.streamEntries(this.cfg.table, limit).
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
},
|
|
236
|
-
{ concurrency: 32 },
|
|
237
|
-
)
|
|
214
|
+
return this.cfg.db.streamEntries(this.cfg.table, limit).mapSync(([id, buf]) => {
|
|
215
|
+
try {
|
|
216
|
+
return [id as K, transformer.bufferToValue(buf)]
|
|
217
|
+
} catch (err) {
|
|
218
|
+
this.cfg.logger.error(err)
|
|
219
|
+
return SKIP
|
|
220
|
+
}
|
|
221
|
+
})
|
|
238
222
|
}
|
|
239
223
|
|
|
240
224
|
async getAllKeys(limit?: number): Promise<K[]> {
|