@naturalcycles/db-lib 10.47.1 → 10.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -143,11 +143,11 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
143
143
  * const storageRow = await dao.dbmToStorageRow(dbm)
144
144
  * await db.saveBatch(table, [storageRow])
145
145
  */
146
- dbmToStorageRow(dbm: DBM): Promise<ObjectWithId>;
146
+ dbmToStorageRow(dbm: DBM): ObjectWithId;
147
147
  /**
148
148
  * Converts multiple DBMs to storage rows.
149
149
  */
150
- dbmsToStorageRows(dbms: DBM[]): Promise<ObjectWithId[]>;
150
+ dbmsToStorageRows(dbms: DBM[]): ObjectWithId[];
151
151
  /**
152
152
  * Converts a storage row back to a DBM, applying decompression if needed.
153
153
  *
@@ -157,12 +157,18 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
157
157
  * const rows = await db.getByIds(table, ids)
158
158
  * const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
159
159
  */
160
- storageRowToDBM(row: ObjectWithId): Promise<DBM>;
160
+ storageRowToDBM(row: ObjectWithId): DBM;
161
161
  /**
162
162
  * Converts multiple storage rows to DBMs.
163
163
  */
164
- storageRowsToDBMs(rows: ObjectWithId[]): Promise<DBM[]>;
164
+ storageRowsToDBM(rows: ObjectWithId[]): DBM[];
165
+ /**
166
+ * Mutates `dbm`.
167
+ */
165
168
  private compress;
169
+ /**
170
+ * Mutates `dbm`.
171
+ */
166
172
  private decompress;
167
173
  anyToDBM(dbm: undefined, opt?: CommonDaoOptions): null;
168
174
  anyToDBM(dbm?: any, opt?: CommonDaoOptions): DBM;
@@ -7,7 +7,7 @@ import { _filterUndefinedValues, _objectAssignExact, _omitWithUndefined, _pick,
7
7
  import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
8
8
  import { _objectKeys, _passthroughPredicate, _stringMapEntries, _stringMapValues, _typeCast, } from '@naturalcycles/js-lib/types';
9
9
  import { stringId } from '@naturalcycles/nodejs-lib';
10
- import { decompressZstdOrInflateToString, zstdCompress } from '@naturalcycles/nodejs-lib/zip';
10
+ import { zip2 } from '@naturalcycles/nodejs-lib/zip';
11
11
  import { DBLibError } from '../cnst.js';
12
12
  import { RunnableDBQuery } from '../query/dbQuery.js';
13
13
  import { CommonDaoTransaction } from './commonDaoTransaction.js';
@@ -105,7 +105,7 @@ export class CommonDao {
105
105
  return [];
106
106
  const table = opt.table || this.cfg.table;
107
107
  const rows = await (opt.tx || this.cfg.db).getByIds(table, ids, opt);
108
- return await this.storageRowsToDBMs(rows);
108
+ return this.storageRowsToDBM(rows);
109
109
  }
110
110
  async getBy(by, value, limit = 0, opt) {
111
111
  return await this.query().filterEq(by, value).limit(limit).runQuery(opt);
@@ -148,7 +148,7 @@ export class CommonDao {
148
148
  q.table = opt.table || q.table;
149
149
  const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
150
150
  const isPartialQuery = !!q._selectedFieldNames;
151
- const rows = isPartialQuery ? rawRows : await this.storageRowsToDBMs(rawRows);
151
+ const rows = isPartialQuery ? rawRows : this.storageRowsToDBM(rawRows);
152
152
  const bms = isPartialQuery ? rows : this.dbmsToBM(rows, opt);
153
153
  return {
154
154
  rows: bms,
@@ -164,7 +164,7 @@ export class CommonDao {
164
164
  q.table = opt.table || q.table;
165
165
  const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery(q, opt);
166
166
  const isPartialQuery = !!q._selectedFieldNames;
167
- const rows = isPartialQuery ? rawRows : await this.storageRowsToDBMs(rawRows);
167
+ const rows = isPartialQuery ? rawRows : this.storageRowsToDBM(rawRows);
168
168
  const dbms = isPartialQuery ? rows : this.anyToDBMs(rows, opt);
169
169
  return { rows: dbms, ...queryResult };
170
170
  }
@@ -178,10 +178,7 @@ export class CommonDao {
178
178
  q.table = opt.table || q.table;
179
179
  let pipeline = this.cfg.db.streamQuery(q, opt);
180
180
  if (this.cfg.compress?.keys.length) {
181
- pipeline = pipeline.map(async row => await this.storageRowToDBM(row), {
182
- // lowered, to reduce the total buffer size a bit for uncompressed objects
183
- highWaterMark: 16,
184
- });
181
+ pipeline = pipeline.mapSync(row => this.storageRowToDBM(row));
185
182
  }
186
183
  const isPartialQuery = !!q._selectedFieldNames;
187
184
  if (isPartialQuery)
@@ -195,10 +192,7 @@ export class CommonDao {
195
192
  q.table = opt.table || q.table;
196
193
  let pipeline = this.cfg.db.streamQuery(q, opt);
197
194
  if (this.cfg.compress?.keys.length) {
198
- pipeline = pipeline.map(async row => await this.storageRowToDBM(row), {
199
- // lowered, to reduce the total buffer size a bit for uncompressed objects
200
- highWaterMark: 16,
201
- });
195
+ pipeline = pipeline.mapSync(row => this.storageRowToDBM(row));
202
196
  }
203
197
  const isPartialQuery = !!q._selectedFieldNames;
204
198
  if (isPartialQuery)
@@ -357,7 +351,7 @@ export class CommonDao {
357
351
  this.cfg.hooks.beforeSave?.(dbm);
358
352
  const table = opt.table || this.cfg.table;
359
353
  const saveOptions = this.prepareSaveOptions(opt);
360
- const row = await this.dbmToStorageRow(dbm);
354
+ const row = this.dbmToStorageRow(dbm);
361
355
  await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions);
362
356
  if (saveOptions.assignGeneratedIds) {
363
357
  bm.id = dbm.id;
@@ -371,7 +365,7 @@ export class CommonDao {
371
365
  this.cfg.hooks.beforeSave?.(validDbm);
372
366
  const table = opt.table || this.cfg.table;
373
367
  const saveOptions = this.prepareSaveOptions(opt);
374
- const row = await this.dbmToStorageRow(validDbm);
368
+ const row = this.dbmToStorageRow(validDbm);
375
369
  await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions);
376
370
  if (saveOptions.assignGeneratedIds) {
377
371
  dbm.id = validDbm.id;
@@ -389,7 +383,7 @@ export class CommonDao {
389
383
  }
390
384
  const table = opt.table || this.cfg.table;
391
385
  const saveOptions = this.prepareSaveOptions(opt);
392
- const rows = await this.dbmsToStorageRows(dbms);
386
+ const rows = this.dbmsToStorageRows(dbms);
393
387
  await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions);
394
388
  if (saveOptions.assignGeneratedIds) {
395
389
  dbms.forEach((dbm, i) => (bms[i].id = dbm.id));
@@ -407,7 +401,7 @@ export class CommonDao {
407
401
  }
408
402
  const table = opt.table || this.cfg.table;
409
403
  const saveOptions = this.prepareSaveOptions(opt);
410
- const rows = await this.dbmsToStorageRows(validDbms);
404
+ const rows = this.dbmsToStorageRows(validDbms);
411
405
  await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions);
412
406
  if (saveOptions.assignGeneratedIds) {
413
407
  validDbms.forEach((dbm, i) => (dbms[i].id = dbm.id));
@@ -457,7 +451,7 @@ export class CommonDao {
457
451
  this.assignIdCreatedUpdated(bm, opt);
458
452
  const dbm = this.bmToDBM(bm, opt);
459
453
  beforeSave?.(dbm);
460
- return await this.dbmToStorageRow(dbm);
454
+ return this.dbmToStorageRow(dbm);
461
455
  }, { errorMode })
462
456
  .chunk(chunkSize)
463
457
  .map(async batch => {
@@ -606,20 +600,24 @@ export class CommonDao {
606
600
  * const storageRow = await dao.dbmToStorageRow(dbm)
607
601
  * await db.saveBatch(table, [storageRow])
608
602
  */
609
- async dbmToStorageRow(dbm) {
603
+ dbmToStorageRow(dbm) {
610
604
  if (!this.cfg.compress?.keys.length)
611
605
  return dbm;
612
606
  const row = { ...dbm };
613
- await this.compress(row);
607
+ this.compress(row);
614
608
  return row;
615
609
  }
616
610
  /**
617
611
  * Converts multiple DBMs to storage rows.
618
612
  */
619
- async dbmsToStorageRows(dbms) {
613
+ dbmsToStorageRows(dbms) {
620
614
  if (!this.cfg.compress?.keys.length)
621
615
  return dbms;
622
- return await pMap(dbms, async dbm => await this.dbmToStorageRow(dbm));
616
+ return dbms.map(dbm => {
617
+ const row = { ...dbm };
618
+ this.compress(row);
619
+ return row;
620
+ });
623
621
  }
624
622
  /**
625
623
  * Converts a storage row back to a DBM, applying decompression if needed.
@@ -630,43 +628,46 @@ export class CommonDao {
630
628
  * const rows = await db.getByIds(table, ids)
631
629
  * const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
632
630
  */
633
- async storageRowToDBM(row) {
631
+ storageRowToDBM(row) {
634
632
  if (!this.cfg.compress?.keys.length)
635
633
  return row;
636
634
  const dbm = { ...row };
637
- await this.decompress(dbm);
635
+ this.decompress(dbm);
638
636
  return dbm;
639
637
  }
640
638
  /**
641
639
  * Converts multiple storage rows to DBMs.
642
640
  */
643
- async storageRowsToDBMs(rows) {
641
+ storageRowsToDBM(rows) {
644
642
  if (!this.cfg.compress?.keys.length)
645
643
  return rows;
646
- return await pMap(rows, async row => await this.storageRowToDBM(row));
644
+ return rows.map(row => {
645
+ const dbm = { ...row };
646
+ this.decompress(dbm);
647
+ return dbm;
648
+ });
647
649
  }
648
650
  /**
649
651
  * Mutates `dbm`.
650
652
  */
651
- async compress(dbm) {
653
+ compress(dbm) {
652
654
  if (!this.cfg.compress?.keys.length)
653
655
  return; // No compression requested
654
656
  const { keys, level = 1 } = this.cfg.compress;
655
657
  const properties = _pick(dbm, keys);
656
658
  const bufferString = JSON.stringify(properties);
657
- const __compressed = await zstdCompress(bufferString, level);
659
+ const __compressed = zip2.zstdCompressSync(bufferString, level);
658
660
  _omitWithUndefined(dbm, _objectKeys(properties), { mutate: true });
659
661
  Object.assign(dbm, { __compressed });
660
662
  }
661
663
  /**
662
664
  * Mutates `dbm`.
663
665
  */
664
- async decompress(dbm) {
666
+ decompress(dbm) {
665
667
  _typeCast(dbm);
666
668
  if (!Buffer.isBuffer(dbm.__compressed))
667
669
  return; // No compressed data
668
- // todo: stop supporting Inflate when we are sure that we have migrated everything to zstd
669
- const bufferString = await decompressZstdOrInflateToString(dbm.__compressed);
670
+ const bufferString = zip2.zstdDecompressToStringSync(dbm.__compressed);
670
671
  const properties = JSON.parse(bufferString);
671
672
  dbm.__compressed = undefined;
672
673
  Object.assign(dbm, properties);
@@ -769,7 +770,7 @@ export class CommonDao {
769
770
  // todo: support tx
770
771
  const dbmsByTable = await db.multiGet(idsByTable, opt);
771
772
  const dbmByTableById = CommonDao.multiGetMapByTableById(dbmsByTable);
772
- return (await CommonDao.prepareMultiGetOutput(inputMap, dbmByTableById, opt));
773
+ return CommonDao.prepareMultiGetOutput(inputMap, dbmByTableById, opt);
773
774
  }
774
775
  static prepareMultiGetIds(inputMap) {
775
776
  const idSetByTable = {};
@@ -806,17 +807,17 @@ export class CommonDao {
806
807
  }
807
808
  return dbmByTableById;
808
809
  }
809
- static async prepareMultiGetOutput(inputMap, dbmByTableById, opt = {}) {
810
+ static prepareMultiGetOutput(inputMap, dbmByTableById, opt = {}) {
810
811
  const bmsByProp = {};
811
812
  // Loop over input props again, to produce the output of the same shape as requested
812
- await pMap(_stringMapEntries(inputMap), async ([prop, input]) => {
813
+ for (const [prop, input] of _stringMapEntries(inputMap)) {
813
814
  const { dao } = input;
814
815
  const { table } = dao.cfg;
815
816
  if ('id' in input) {
816
817
  // Singular
817
818
  const row = dbmByTableById[table][input.id];
818
819
  // Decompress before converting to BM
819
- const dbm = row ? await dao.storageRowToDBM(row) : undefined;
820
+ const dbm = row ? dao.storageRowToDBM(row) : undefined;
820
821
  bmsByProp[prop] = dao.dbmToBM(dbm, opt) || null;
821
822
  }
822
823
  else {
@@ -825,10 +826,10 @@ export class CommonDao {
825
826
  // Without filtering - every prop will get ALL rows from that table.
826
827
  const rows = input.ids.map(id => dbmByTableById[table][id]).filter(_isTruthy);
827
828
  // Decompress before converting to BM
828
- const dbms = await dao.storageRowsToDBMs(rows);
829
+ const dbms = dao.storageRowsToDBM(rows);
829
830
  bmsByProp[prop] = dao.dbmsToBM(dbms, opt);
830
831
  }
831
- });
832
+ }
832
833
  return bmsByProp;
833
834
  }
834
835
  /**
@@ -859,7 +860,7 @@ export class CommonDao {
859
860
  return;
860
861
  const { db } = inputs[0].dao.cfg;
861
862
  const dbmsByTable = {};
862
- await pMap(inputs, async input => {
863
+ for (const input of inputs) {
863
864
  const { dao } = input;
864
865
  const { table } = dao.cfg;
865
866
  dbmsByTable[table] ||= [];
@@ -879,7 +880,7 @@ export class CommonDao {
879
880
  dao.assignIdCreatedUpdated(row, opt);
880
881
  const dbm = dao.bmToDBM(row, opt);
881
882
  dao.cfg.hooks.beforeSave?.(dbm);
882
- const storageRow = await dao.dbmToStorageRow(dbm);
883
+ const storageRow = dao.dbmToStorageRow(dbm);
883
884
  dbmsByTable[table].push(storageRow);
884
885
  }
885
886
  else {
@@ -889,10 +890,10 @@ export class CommonDao {
889
890
  if (dao.cfg.hooks.beforeSave) {
890
891
  dbms.forEach(dbm => dao.cfg.hooks.beforeSave(dbm));
891
892
  }
892
- const storageRows = await dao.dbmsToStorageRows(dbms);
893
+ const storageRows = dao.dbmsToStorageRows(dbms);
893
894
  dbmsByTable[table].push(...storageRows);
894
895
  }
895
- });
896
+ }
896
897
  await db.multiSave(dbmsByTable);
897
898
  }
898
899
  async createTransaction(opt) {
@@ -28,8 +28,8 @@ export interface CommonKeyValueDaoCfg<V> {
28
28
  }
29
29
  export type CommonKeyValueDaoSaveOptions = CommonKeyValueDBSaveBatchOptions;
30
30
  export interface CommonKeyValueDaoTransformer<V> {
31
- valueToBuffer: (v: V) => Promise<Buffer>;
32
- bufferToValue: (buf: Buffer) => Promise<V>;
31
+ valueToBuffer: (v: V) => Buffer;
32
+ bufferToValue: (buf: Buffer) => V;
33
33
  }
34
34
  /**
35
35
  * @deprecated use zstd instead, gzip is obsolete
@@ -1,20 +1,19 @@
1
1
  import { AppError } from '@naturalcycles/js-lib/error/error.util.js';
2
- import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
3
2
  import { SKIP } from '@naturalcycles/js-lib/types';
4
- import { decompressZstdOrInflateToString, deflateString, inflateToString, zstdCompress, zstdDecompressToString, } from '@naturalcycles/nodejs-lib/zip';
3
+ import { zip2 } from '@naturalcycles/nodejs-lib/zip';
5
4
  /**
6
5
  * @deprecated use zstd instead, gzip is obsolete
7
6
  */
8
7
  export function commonKeyValueDaoDeflatedJsonTransformer() {
9
8
  return {
10
- valueToBuffer: async v => await deflateString(JSON.stringify(v)),
11
- bufferToValue: async buf => JSON.parse(await inflateToString(buf)),
9
+ valueToBuffer: v => zip2.deflateSync(JSON.stringify(v)),
10
+ bufferToValue: buf => JSON.parse(zip2.inflateToStringSync(buf)),
12
11
  };
13
12
  }
14
13
  export function commonKeyValueDaoZstdJsonTransformer(level) {
15
14
  return {
16
- valueToBuffer: async v => await zstdCompress(JSON.stringify(v), level),
17
- bufferToValue: async buf => JSON.parse(await zstdDecompressToString(buf)),
15
+ valueToBuffer: v => zip2.zstdCompressSync(JSON.stringify(v), level),
16
+ bufferToValue: buf => JSON.parse(zip2.zstdDecompressToStringSync(buf)),
18
17
  };
19
18
  }
20
19
  /**
@@ -23,8 +22,8 @@ export function commonKeyValueDaoZstdJsonTransformer(level) {
23
22
  */
24
23
  export function commonKeyValueDaoCompressedTransformer() {
25
24
  return {
26
- valueToBuffer: async v => await zstdCompress(JSON.stringify(v)),
27
- bufferToValue: async buf => JSON.parse(await decompressZstdOrInflateToString(buf)),
25
+ valueToBuffer: v => zip2.zstdCompressSync(JSON.stringify(v)),
26
+ bufferToValue: buf => JSON.parse(zip2.decompressZstdOrInflateToStringSync(buf)),
28
27
  };
29
28
  }
30
29
  // todo: logging
@@ -81,10 +80,7 @@ export class CommonKeyValueDao {
81
80
  const entries = await this.cfg.db.getByIds(this.cfg.table, ids);
82
81
  if (!this.cfg.transformer)
83
82
  return entries;
84
- return await pMap(entries, async ([id, raw]) => [
85
- id,
86
- await this.cfg.transformer.bufferToValue(raw),
87
- ]);
83
+ return entries.map(([id, raw]) => [id, this.cfg.transformer.bufferToValue(raw)]);
88
84
  }
89
85
  async getByIdsAsBuffer(ids) {
90
86
  return await this.cfg.db.getByIds(this.cfg.table, ids);
@@ -99,7 +95,7 @@ export class CommonKeyValueDao {
99
95
  rawEntries = entries;
100
96
  }
101
97
  else {
102
- rawEntries = await pMap(entries, async ([id, v]) => [id, await transformer.valueToBuffer(v)]);
98
+ rawEntries = entries.map(([id, v]) => [id, transformer.valueToBuffer(v)]);
103
99
  }
104
100
  await this.cfg.db.saveBatch(this.cfg.table, rawEntries, opt);
105
101
  }
@@ -117,30 +113,30 @@ export class CommonKeyValueDao {
117
113
  if (!transformer) {
118
114
  return this.cfg.db.streamValues(this.cfg.table, limit);
119
115
  }
120
- return this.cfg.db.streamValues(this.cfg.table, limit).map(async buf => {
116
+ return this.cfg.db.streamValues(this.cfg.table, limit).mapSync(buf => {
121
117
  try {
122
- return await transformer.bufferToValue(buf);
118
+ return transformer.bufferToValue(buf);
123
119
  }
124
120
  catch (err) {
125
121
  this.cfg.logger.error(err);
126
122
  return SKIP;
127
123
  }
128
- }, { concurrency: 32 });
124
+ });
129
125
  }
130
126
  streamEntries(limit) {
131
127
  const { transformer } = this.cfg;
132
128
  if (!transformer) {
133
129
  return this.cfg.db.streamEntries(this.cfg.table, limit);
134
130
  }
135
- return this.cfg.db.streamEntries(this.cfg.table, limit).map(async ([id, buf]) => {
131
+ return this.cfg.db.streamEntries(this.cfg.table, limit).mapSync(([id, buf]) => {
136
132
  try {
137
- return [id, await transformer.bufferToValue(buf)];
133
+ return [id, transformer.bufferToValue(buf)];
138
134
  }
139
135
  catch (err) {
140
136
  this.cfg.logger.error(err);
141
137
  return SKIP;
142
138
  }
143
- }, { concurrency: 32 });
139
+ });
144
140
  }
145
141
  async getAllKeys(limit) {
146
142
  return await this.streamIds(limit).toArray();
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@naturalcycles/db-lib",
3
3
  "type": "module",
4
- "version": "10.47.1",
4
+ "version": "10.49.0",
5
5
  "dependencies": {
6
6
  "@naturalcycles/js-lib": "^15",
7
7
  "@naturalcycles/nodejs-lib": "^15"
@@ -27,7 +27,7 @@ import type {
27
27
  import { stringId } from '@naturalcycles/nodejs-lib'
28
28
  import type { JsonSchema } from '@naturalcycles/nodejs-lib/ajv'
29
29
  import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
30
- import { decompressZstdOrInflateToString, zstdCompress } from '@naturalcycles/nodejs-lib/zip'
30
+ import { zip2 } from '@naturalcycles/nodejs-lib/zip'
31
31
  import { DBLibError } from '../cnst.js'
32
32
  import type {
33
33
  CommonDBSaveOptions,
@@ -155,7 +155,7 @@ export class CommonDao<
155
155
  if (!ids.length) return []
156
156
  const table = opt.table || this.cfg.table
157
157
  const rows = await (opt.tx || this.cfg.db).getByIds<DBM>(table, ids, opt)
158
- return await this.storageRowsToDBMs(rows)
158
+ return this.storageRowsToDBM(rows)
159
159
  }
160
160
 
161
161
  async getBy(by: keyof DBM, value: any, limit = 0, opt?: CommonDaoReadOptions): Promise<BM[]> {
@@ -216,7 +216,7 @@ export class CommonDao<
216
216
  q.table = opt.table || q.table
217
217
  const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
218
218
  const isPartialQuery = !!q._selectedFieldNames
219
- const rows = isPartialQuery ? rawRows : await this.storageRowsToDBMs(rawRows)
219
+ const rows = isPartialQuery ? rawRows : this.storageRowsToDBM(rawRows)
220
220
  const bms = isPartialQuery ? (rows as any[]) : this.dbmsToBM(rows, opt)
221
221
  return {
222
222
  rows: bms,
@@ -237,7 +237,7 @@ export class CommonDao<
237
237
  q.table = opt.table || q.table
238
238
  const { rows: rawRows, ...queryResult } = await this.cfg.db.runQuery<DBM>(q, opt)
239
239
  const isPartialQuery = !!q._selectedFieldNames
240
- const rows = isPartialQuery ? rawRows : await this.storageRowsToDBMs(rawRows)
240
+ const rows = isPartialQuery ? rawRows : this.storageRowsToDBM(rawRows)
241
241
  const dbms = isPartialQuery ? rows : this.anyToDBMs(rows, opt)
242
242
  return { rows: dbms, ...queryResult }
243
243
  }
@@ -254,10 +254,7 @@ export class CommonDao<
254
254
  let pipeline = this.cfg.db.streamQuery<DBM>(q, opt)
255
255
 
256
256
  if (this.cfg.compress?.keys.length) {
257
- pipeline = pipeline.map(async row => await this.storageRowToDBM(row), {
258
- // lowered, to reduce the total buffer size a bit for uncompressed objects
259
- highWaterMark: 16,
260
- })
257
+ pipeline = pipeline.mapSync(row => this.storageRowToDBM(row))
261
258
  }
262
259
 
263
260
  const isPartialQuery = !!q._selectedFieldNames
@@ -275,10 +272,7 @@ export class CommonDao<
275
272
  let pipeline = this.cfg.db.streamQuery<DBM>(q, opt)
276
273
 
277
274
  if (this.cfg.compress?.keys.length) {
278
- pipeline = pipeline.map(async row => await this.storageRowToDBM(row), {
279
- // lowered, to reduce the total buffer size a bit for uncompressed objects
280
- highWaterMark: 16,
281
- })
275
+ pipeline = pipeline.mapSync(row => this.storageRowToDBM(row))
282
276
  }
283
277
 
284
278
  const isPartialQuery = !!q._selectedFieldNames
@@ -477,7 +471,7 @@ export class CommonDao<
477
471
  const table = opt.table || this.cfg.table
478
472
  const saveOptions = this.prepareSaveOptions(opt)
479
473
 
480
- const row = await this.dbmToStorageRow(dbm)
474
+ const row = this.dbmToStorageRow(dbm)
481
475
  await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions)
482
476
 
483
477
  if (saveOptions.assignGeneratedIds) {
@@ -495,7 +489,7 @@ export class CommonDao<
495
489
  const table = opt.table || this.cfg.table
496
490
  const saveOptions = this.prepareSaveOptions(opt)
497
491
 
498
- const row = await this.dbmToStorageRow(validDbm)
492
+ const row = this.dbmToStorageRow(validDbm)
499
493
  await (opt.tx || this.cfg.db).saveBatch(table, [row], saveOptions)
500
494
 
501
495
  if (saveOptions.assignGeneratedIds) {
@@ -516,7 +510,7 @@ export class CommonDao<
516
510
  const table = opt.table || this.cfg.table
517
511
  const saveOptions = this.prepareSaveOptions(opt)
518
512
 
519
- const rows = await this.dbmsToStorageRows(dbms)
513
+ const rows = this.dbmsToStorageRows(dbms)
520
514
  await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions)
521
515
 
522
516
  if (saveOptions.assignGeneratedIds) {
@@ -540,7 +534,7 @@ export class CommonDao<
540
534
  const table = opt.table || this.cfg.table
541
535
  const saveOptions = this.prepareSaveOptions(opt)
542
536
 
543
- const rows = await this.dbmsToStorageRows(validDbms)
537
+ const rows = this.dbmsToStorageRows(validDbms)
544
538
  await (opt.tx || this.cfg.db).saveBatch(table, rows, saveOptions)
545
539
 
546
540
  if (saveOptions.assignGeneratedIds) {
@@ -608,7 +602,7 @@ export class CommonDao<
608
602
  this.assignIdCreatedUpdated(bm, opt)
609
603
  const dbm = this.bmToDBM(bm, opt)
610
604
  beforeSave?.(dbm)
611
- return await this.dbmToStorageRow(dbm)
605
+ return this.dbmToStorageRow(dbm)
612
606
  },
613
607
  { errorMode },
614
608
  )
@@ -802,19 +796,23 @@ export class CommonDao<
802
796
  * const storageRow = await dao.dbmToStorageRow(dbm)
803
797
  * await db.saveBatch(table, [storageRow])
804
798
  */
805
- async dbmToStorageRow(dbm: DBM): Promise<ObjectWithId> {
799
+ dbmToStorageRow(dbm: DBM): ObjectWithId {
806
800
  if (!this.cfg.compress?.keys.length) return dbm
807
801
  const row = { ...dbm }
808
- await this.compress(row)
802
+ this.compress(row)
809
803
  return row
810
804
  }
811
805
 
812
806
  /**
813
807
  * Converts multiple DBMs to storage rows.
814
808
  */
815
- async dbmsToStorageRows(dbms: DBM[]): Promise<ObjectWithId[]> {
809
+ dbmsToStorageRows(dbms: DBM[]): ObjectWithId[] {
816
810
  if (!this.cfg.compress?.keys.length) return dbms
817
- return await pMap(dbms, async dbm => await this.dbmToStorageRow(dbm))
811
+ return dbms.map(dbm => {
812
+ const row = { ...dbm }
813
+ this.compress(row)
814
+ return row
815
+ })
818
816
  }
819
817
 
820
818
  /**
@@ -826,31 +824,35 @@ export class CommonDao<
826
824
  * const rows = await db.getByIds(table, ids)
827
825
  * const dbms = await Promise.all(rows.map(row => dao.storageRowToDBM(row)))
828
826
  */
829
- async storageRowToDBM(row: ObjectWithId): Promise<DBM> {
827
+ storageRowToDBM(row: ObjectWithId): DBM {
830
828
  if (!this.cfg.compress?.keys.length) return row as DBM
831
829
  const dbm = { ...(row as DBM) }
832
- await this.decompress(dbm)
830
+ this.decompress(dbm)
833
831
  return dbm
834
832
  }
835
833
 
836
834
  /**
837
835
  * Converts multiple storage rows to DBMs.
838
836
  */
839
- async storageRowsToDBMs(rows: ObjectWithId[]): Promise<DBM[]> {
837
+ storageRowsToDBM(rows: ObjectWithId[]): DBM[] {
840
838
  if (!this.cfg.compress?.keys.length) return rows as DBM[]
841
- return await pMap(rows, async row => await this.storageRowToDBM(row))
839
+ return rows.map(row => {
840
+ const dbm = { ...(row as DBM) }
841
+ this.decompress(dbm)
842
+ return dbm
843
+ })
842
844
  }
843
845
 
844
846
  /**
845
847
  * Mutates `dbm`.
846
848
  */
847
- private async compress(dbm: DBM): Promise<void> {
849
+ private compress(dbm: DBM): void {
848
850
  if (!this.cfg.compress?.keys.length) return // No compression requested
849
851
 
850
852
  const { keys, level = 1 } = this.cfg.compress
851
853
  const properties = _pick(dbm, keys)
852
854
  const bufferString = JSON.stringify(properties)
853
- const __compressed = await zstdCompress(bufferString, level)
855
+ const __compressed = zip2.zstdCompressSync(bufferString, level)
854
856
  _omitWithUndefined(dbm as any, _objectKeys(properties), { mutate: true })
855
857
  Object.assign(dbm, { __compressed })
856
858
  }
@@ -858,12 +860,11 @@ export class CommonDao<
858
860
  /**
859
861
  * Mutates `dbm`.
860
862
  */
861
- private async decompress(dbm: DBM): Promise<void> {
863
+ private decompress(dbm: DBM): void {
862
864
  _typeCast<Compressed<DBM>>(dbm)
863
865
  if (!Buffer.isBuffer(dbm.__compressed)) return // No compressed data
864
866
 
865
- // todo: stop supporting Inflate when we are sure that we have migrated everything to zstd
866
- const bufferString = await decompressZstdOrInflateToString(dbm.__compressed)
867
+ const bufferString = zip2.zstdDecompressToStringSync(dbm.__compressed)
867
868
  const properties = JSON.parse(bufferString)
868
869
  dbm.__compressed = undefined
869
870
  Object.assign(dbm, properties)
@@ -1001,7 +1002,7 @@ export class CommonDao<
1001
1002
 
1002
1003
  const dbmByTableById = CommonDao.multiGetMapByTableById(dbmsByTable)
1003
1004
 
1004
- return (await CommonDao.prepareMultiGetOutput(inputMap, dbmByTableById, opt)) as any
1005
+ return CommonDao.prepareMultiGetOutput(inputMap, dbmByTableById, opt) as any
1005
1006
  }
1006
1007
 
1007
1008
  private static prepareMultiGetIds(
@@ -1047,22 +1048,22 @@ export class CommonDao<
1047
1048
  return dbmByTableById
1048
1049
  }
1049
1050
 
1050
- private static async prepareMultiGetOutput(
1051
+ private static prepareMultiGetOutput(
1051
1052
  inputMap: StringMap<DaoWithIds<AnyDao> | DaoWithId<AnyDao>>,
1052
1053
  dbmByTableById: StringMap<StringMap<ObjectWithId>>,
1053
1054
  opt: CommonDaoReadOptions = {},
1054
- ): Promise<StringMap<unknown>> {
1055
+ ): StringMap<unknown> {
1055
1056
  const bmsByProp: StringMap<unknown> = {}
1056
1057
 
1057
1058
  // Loop over input props again, to produce the output of the same shape as requested
1058
- await pMap(_stringMapEntries(inputMap), async ([prop, input]) => {
1059
+ for (const [prop, input] of _stringMapEntries(inputMap)) {
1059
1060
  const { dao } = input
1060
1061
  const { table } = dao.cfg
1061
1062
  if ('id' in input) {
1062
1063
  // Singular
1063
1064
  const row = dbmByTableById[table]![input.id]
1064
1065
  // Decompress before converting to BM
1065
- const dbm = row ? await dao.storageRowToDBM(row) : undefined
1066
+ const dbm = row ? dao.storageRowToDBM(row) : undefined
1066
1067
  bmsByProp[prop] = dao.dbmToBM(dbm, opt) || null
1067
1068
  } else {
1068
1069
  // Plural
@@ -1070,10 +1071,10 @@ export class CommonDao<
1070
1071
  // Without filtering - every prop will get ALL rows from that table.
1071
1072
  const rows = input.ids.map(id => dbmByTableById[table]![id]).filter(_isTruthy)
1072
1073
  // Decompress before converting to BM
1073
- const dbms = await dao.storageRowsToDBMs(rows)
1074
+ const dbms = dao.storageRowsToDBM(rows)
1074
1075
  bmsByProp[prop] = dao.dbmsToBM(dbms, opt)
1075
1076
  }
1076
- })
1077
+ }
1077
1078
 
1078
1079
  return bmsByProp as any
1079
1080
  }
@@ -1112,7 +1113,7 @@ export class CommonDao<
1112
1113
  if (!inputs.length) return
1113
1114
  const { db } = inputs[0]!.dao.cfg
1114
1115
  const dbmsByTable: StringMap<any[]> = {}
1115
- await pMap(inputs, async input => {
1116
+ for (const input of inputs) {
1116
1117
  const { dao } = input
1117
1118
  const { table } = dao.cfg
1118
1119
  dbmsByTable[table] ||= []
@@ -1135,7 +1136,7 @@ export class CommonDao<
1135
1136
  dao.assignIdCreatedUpdated(row, opt)
1136
1137
  const dbm = dao.bmToDBM(row, opt)
1137
1138
  dao.cfg.hooks!.beforeSave?.(dbm)
1138
- const storageRow = await dao.dbmToStorageRow(dbm)
1139
+ const storageRow = dao.dbmToStorageRow(dbm)
1139
1140
  dbmsByTable[table].push(storageRow)
1140
1141
  } else {
1141
1142
  // Plural
@@ -1144,10 +1145,10 @@ export class CommonDao<
1144
1145
  if (dao.cfg.hooks!.beforeSave) {
1145
1146
  dbms.forEach(dbm => dao.cfg.hooks!.beforeSave!(dbm))
1146
1147
  }
1147
- const storageRows = await dao.dbmsToStorageRows(dbms)
1148
+ const storageRows = dao.dbmsToStorageRows(dbms)
1148
1149
  dbmsByTable[table].push(...storageRows)
1149
1150
  }
1150
- })
1151
+ }
1151
1152
 
1152
1153
  await db.multiSave(dbmsByTable)
1153
1154
  }
@@ -1,16 +1,9 @@
1
1
  import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
2
2
  import type { CommonLogger } from '@naturalcycles/js-lib/log'
3
- import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
4
3
  import { SKIP } from '@naturalcycles/js-lib/types'
5
4
  import type { Integer, KeyValueTuple } from '@naturalcycles/js-lib/types'
6
5
  import type { Pipeline } from '@naturalcycles/nodejs-lib/stream'
7
- import {
8
- decompressZstdOrInflateToString,
9
- deflateString,
10
- inflateToString,
11
- zstdCompress,
12
- zstdDecompressToString,
13
- } from '@naturalcycles/nodejs-lib/zip'
6
+ import { zip2 } from '@naturalcycles/nodejs-lib/zip'
14
7
  import type { CommonDaoLogLevel } from '../commondao/common.dao.model.js'
15
8
  import type { CommonDBCreateOptions } from '../db.model.js'
16
9
  import type {
@@ -52,8 +45,8 @@ export interface CommonKeyValueDaoCfg<V> {
52
45
  export type CommonKeyValueDaoSaveOptions = CommonKeyValueDBSaveBatchOptions
53
46
 
54
47
  export interface CommonKeyValueDaoTransformer<V> {
55
- valueToBuffer: (v: V) => Promise<Buffer>
56
- bufferToValue: (buf: Buffer) => Promise<V>
48
+ valueToBuffer: (v: V) => Buffer
49
+ bufferToValue: (buf: Buffer) => V
57
50
  }
58
51
 
59
52
  /**
@@ -63,8 +56,8 @@ export function commonKeyValueDaoDeflatedJsonTransformer<
63
56
  T = any,
64
57
  >(): CommonKeyValueDaoTransformer<T> {
65
58
  return {
66
- valueToBuffer: async v => await deflateString(JSON.stringify(v)),
67
- bufferToValue: async buf => JSON.parse(await inflateToString(buf)),
59
+ valueToBuffer: v => zip2.deflateSync(JSON.stringify(v)),
60
+ bufferToValue: buf => JSON.parse(zip2.inflateToStringSync(buf)),
68
61
  }
69
62
  }
70
63
 
@@ -72,8 +65,8 @@ export function commonKeyValueDaoZstdJsonTransformer<T = any>(
72
65
  level: Integer | undefined, // defaults to 3
73
66
  ): CommonKeyValueDaoTransformer<T> {
74
67
  return {
75
- valueToBuffer: async v => await zstdCompress(JSON.stringify(v), level),
76
- bufferToValue: async buf => JSON.parse(await zstdDecompressToString(buf)),
68
+ valueToBuffer: v => zip2.zstdCompressSync(JSON.stringify(v), level),
69
+ bufferToValue: buf => JSON.parse(zip2.zstdDecompressToStringSync(buf)),
77
70
  }
78
71
  }
79
72
 
@@ -83,8 +76,8 @@ export function commonKeyValueDaoZstdJsonTransformer<T = any>(
83
76
  */
84
77
  export function commonKeyValueDaoCompressedTransformer<T = any>(): CommonKeyValueDaoTransformer<T> {
85
78
  return {
86
- valueToBuffer: async v => await zstdCompress(JSON.stringify(v)),
87
- bufferToValue: async buf => JSON.parse(await decompressZstdOrInflateToString(buf)),
79
+ valueToBuffer: v => zip2.zstdCompressSync(JSON.stringify(v)),
80
+ bufferToValue: buf => JSON.parse(zip2.decompressZstdOrInflateToStringSync(buf)),
88
81
  }
89
82
  }
90
83
 
@@ -155,10 +148,7 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
155
148
  const entries = await this.cfg.db.getByIds(this.cfg.table, ids)
156
149
  if (!this.cfg.transformer) return entries as any
157
150
 
158
- return await pMap(entries, async ([id, raw]) => [
159
- id,
160
- await this.cfg.transformer!.bufferToValue(raw),
161
- ])
151
+ return entries.map(([id, raw]) => [id, this.cfg.transformer!.bufferToValue(raw)])
162
152
  }
163
153
 
164
154
  async getByIdsAsBuffer(ids: K[]): Promise<KeyValueDBTuple[]> {
@@ -179,7 +169,7 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
179
169
  if (!transformer) {
180
170
  rawEntries = entries as any
181
171
  } else {
182
- rawEntries = await pMap(entries, async ([id, v]) => [id, await transformer.valueToBuffer(v)])
172
+ rawEntries = entries.map(([id, v]) => [id, transformer.valueToBuffer(v)])
183
173
  }
184
174
 
185
175
  await this.cfg.db.saveBatch(this.cfg.table, rawEntries, opt)
@@ -204,17 +194,14 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
204
194
  return this.cfg.db.streamValues(this.cfg.table, limit) as Pipeline<V>
205
195
  }
206
196
 
207
- return this.cfg.db.streamValues(this.cfg.table, limit).map(
208
- async buf => {
209
- try {
210
- return await transformer.bufferToValue(buf)
211
- } catch (err) {
212
- this.cfg.logger.error(err)
213
- return SKIP
214
- }
215
- },
216
- { concurrency: 32 },
217
- )
197
+ return this.cfg.db.streamValues(this.cfg.table, limit).mapSync(buf => {
198
+ try {
199
+ return transformer.bufferToValue(buf)
200
+ } catch (err) {
201
+ this.cfg.logger.error(err)
202
+ return SKIP
203
+ }
204
+ })
218
205
  }
219
206
 
220
207
  streamEntries(limit?: number): Pipeline<KeyValueTuple<K, V>> {
@@ -224,17 +211,14 @@ export class CommonKeyValueDao<K extends string = string, V = Buffer> {
224
211
  return this.cfg.db.streamEntries(this.cfg.table, limit) as Pipeline<KeyValueTuple<K, V>>
225
212
  }
226
213
 
227
- return this.cfg.db.streamEntries(this.cfg.table, limit).map(
228
- async ([id, buf]) => {
229
- try {
230
- return [id as K, await transformer.bufferToValue(buf)]
231
- } catch (err) {
232
- this.cfg.logger.error(err)
233
- return SKIP
234
- }
235
- },
236
- { concurrency: 32 },
237
- )
214
+ return this.cfg.db.streamEntries(this.cfg.table, limit).mapSync(([id, buf]) => {
215
+ try {
216
+ return [id as K, transformer.bufferToValue(buf)]
217
+ } catch (err) {
218
+ this.cfg.logger.error(err)
219
+ return SKIP
220
+ }
221
+ })
238
222
  }
239
223
 
240
224
  async getAllKeys(limit?: number): Promise<K[]> {