@naturalcycles/db-lib 10.23.0 → 10.25.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. package/dist/adapter/file/localFile.persistence.plugin.js +2 -3
  2. package/dist/commondao/common.dao.d.ts +1 -1
  3. package/dist/commondao/common.dao.js +65 -77
  4. package/dist/pipeline/dbPipelineBackup.js +16 -20
  5. package/dist/pipeline/dbPipelineCopy.js +19 -21
  6. package/dist/pipeline/dbPipelineRestore.js +21 -24
  7. package/dist/query/dbQuery.d.ts +4 -1
  8. package/dist/query/dbQuery.js +10 -0
  9. package/dist/testing/commonDaoTest.js +2 -3
  10. package/package.json +2 -2
  11. package/src/adapter/file/localFile.persistence.plugin.ts +2 -7
  12. package/src/commondao/common.dao.ts +32 -45
  13. package/src/pipeline/dbPipelineBackup.ts +10 -21
  14. package/src/pipeline/dbPipelineCopy.ts +12 -22
  15. package/src/pipeline/dbPipelineRestore.ts +14 -27
  16. package/src/query/dbQuery.ts +13 -1
  17. package/src/testing/commonDaoTest.ts +2 -3
package/dist/adapter/file/localFile.persistence.plugin.js CHANGED
@@ -1,7 +1,6 @@
1
- import { Readable } from 'node:stream';
2
1
  import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
3
2
  import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
4
- import { _pipeline, createReadStreamAsNDJSON, createWriteStreamAsNDJSON, } from '@naturalcycles/nodejs-lib/stream';
3
+ import { createReadStreamAsNDJSON, Pipeline } from '@naturalcycles/nodejs-lib/stream';
5
4
  /**
6
5
  * Persists in local filesystem as ndjson.
7
6
  */
@@ -35,6 +34,6 @@ export class LocalFilePersistencePlugin {
35
34
  await fs2.ensureDirAsync(this.cfg.storagePath);
36
35
  const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`;
37
36
  const filePath = `${this.cfg.storagePath}/${table}.${ext}`;
38
- await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(filePath)]);
37
+ await Pipeline.fromArray(rows).toNDJsonFile(filePath);
39
38
  }
40
39
  }
package/dist/commondao/common.dao.d.ts CHANGED
@@ -107,7 +107,7 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
107
107
  * and then executing db.saveBatch(chunk) with the concurrency
108
108
  * of opt.chunkConcurrency (which defaults to 32).
109
109
  */
110
- streamSaveTransform(opt?: CommonDaoStreamSaveOptions<DBM>): Transform[];
110
+ streamSaveTransforms(opt?: CommonDaoStreamSaveOptions<DBM>): Transform[];
111
111
  /**
112
112
  * @returns number of deleted items
113
113
  */
package/dist/commondao/common.dao.js CHANGED
@@ -8,8 +8,8 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
8
8
  import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
9
9
  import { _passthroughPredicate, _typeCast } from '@naturalcycles/js-lib/types';
10
10
  import { stringId } from '@naturalcycles/nodejs-lib';
11
- import { transformFlatten, transformMapSync, } from '@naturalcycles/nodejs-lib/stream';
12
- import { _pipeline, transformChunk, transformLogProgress, transformMap, transformNoOp, writableVoid, } from '@naturalcycles/nodejs-lib/stream';
11
+ import { Pipeline, transformFlatten, transformMapSync, } from '@naturalcycles/nodejs-lib/stream';
12
+ import { transformChunk, transformLogProgress, transformMap, transformNoOp, } from '@naturalcycles/nodejs-lib/stream';
13
13
  import { DBLibError } from '../cnst.js';
14
14
  import { RunnableDBQuery } from '../query/dbQuery.js';
15
15
  import { CommonDaoTransaction } from './commonDaoTransaction.js';
@@ -167,26 +167,24 @@ export class CommonDao {
167
167
  opt.skipValidation = opt.skipValidation !== false; // default true
168
168
  opt.errorMode ||= ErrorMode.SUPPRESS;
169
169
  const isPartialQuery = !!q._selectedFieldNames;
170
- await _pipeline([
171
- this.cfg.db.streamQuery(q, opt),
172
- transformMap(async (dbm) => {
173
- if (isPartialQuery)
174
- return dbm;
175
- return await this.dbmToBM(dbm, opt);
176
- }, {
177
- errorMode: opt.errorMode,
178
- }),
179
- transformMap(mapper, {
180
- ...opt,
181
- predicate: _passthroughPredicate, // to be able to logProgress
182
- }),
170
+ await Pipeline.from(this.cfg.db.streamQuery(q, opt))
171
+ .map(async (dbm) => {
172
+ if (isPartialQuery)
173
+ return dbm;
174
+ return await this.dbmToBM(dbm, opt);
175
+ }, {
176
+ errorMode: opt.errorMode,
177
+ })
178
+ .map(mapper, {
179
+ ...opt,
180
+ predicate: _passthroughPredicate, // to be able to logProgress
181
+ })
183
182
  // LogProgress should be AFTER the mapper, to be able to report correct stats
184
- transformLogProgress({
185
- metric: q.table,
186
- ...opt,
187
- }),
188
- writableVoid(),
189
- ]);
183
+ .logProgress({
184
+ metric: q.table,
185
+ ...opt,
186
+ })
187
+ .run();
190
188
  }
191
189
  async streamQueryAsDBMForEach(q, mapper, opt = {}) {
192
190
  this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
@@ -194,26 +192,23 @@ export class CommonDao {
194
192
  opt.skipValidation = opt.skipValidation !== false; // default true
195
193
  opt.errorMode ||= ErrorMode.SUPPRESS;
196
194
  const isPartialQuery = !!q._selectedFieldNames;
197
- await _pipeline([
198
- this.cfg.db.streamQuery(q, opt),
199
- transformMapSync(dbm => {
200
- if (isPartialQuery)
201
- return dbm;
202
- return this.anyToDBM(dbm, opt);
203
- }, {
204
- errorMode: opt.errorMode,
205
- }),
206
- transformMap(mapper, {
207
- ...opt,
208
- predicate: _passthroughPredicate, // to be able to logProgress
209
- }),
210
- // LogProgress should be AFTER the mapper, to be able to report correct stats
211
- transformLogProgress({
212
- metric: q.table,
213
- ...opt,
214
- }),
215
- writableVoid(),
216
- ]);
195
+ await Pipeline.from(this.cfg.db.streamQuery(q, opt))
196
+ .mapSync(dbm => {
197
+ if (isPartialQuery)
198
+ return dbm;
199
+ return this.anyToDBM(dbm, opt);
200
+ }, {
201
+ errorMode: opt.errorMode,
202
+ })
203
+ .map(mapper, {
204
+ ...opt,
205
+ predicate: _passthroughPredicate, // to be able to logProgress
206
+ })
207
+ .logProgress({
208
+ metric: q.table,
209
+ ...opt,
210
+ })
211
+ .run();
217
212
  }
218
213
  /**
219
214
  * Stream as Readable, to be able to .pipe() it further with support of backpressure.
@@ -306,19 +301,17 @@ export class CommonDao {
306
301
  this.validateQueryIndexes(q); // throws if query uses `excludeFromIndexes` property
307
302
  q.table = opt.table || q.table;
308
303
  opt.errorMode ||= ErrorMode.SUPPRESS;
309
- await _pipeline([
310
- this.cfg.db.streamQuery(q.select(['id']), opt).map(r => r.id),
311
- transformMap(mapper, {
312
- ...opt,
313
- predicate: _passthroughPredicate,
314
- }),
304
+ await Pipeline.from(this.cfg.db.streamQuery(q.select(['id']), opt).map(r => r.id))
305
+ .map(mapper, {
306
+ ...opt,
307
+ predicate: _passthroughPredicate,
308
+ })
315
309
  // LogProgress should be AFTER the mapper, to be able to report correct stats
316
- transformLogProgress({
317
- metric: q.table,
318
- ...opt,
319
- }),
320
- writableVoid(),
321
- ]);
310
+ .logProgress({
311
+ metric: q.table,
312
+ ...opt,
313
+ })
314
+ .run();
322
315
  }
323
316
  /**
324
317
  * Mutates!
@@ -529,7 +522,7 @@ export class CommonDao {
529
522
  * and then executing db.saveBatch(chunk) with the concurrency
530
523
  * of opt.chunkConcurrency (which defaults to 32).
531
524
  */
532
- streamSaveTransform(opt = {}) {
525
+ streamSaveTransforms(opt = {}) {
533
526
  this.requireWriteAccess();
534
527
  const table = opt.table || this.cfg.table;
535
528
  opt.skipValidation ??= true;
@@ -549,7 +542,7 @@ export class CommonDao {
549
542
  }, {
550
543
  errorMode,
551
544
  }),
552
- transformChunk({ chunkSize }),
545
+ transformChunk(chunkSize),
553
546
  transformMap(async (batch) => {
554
547
  await this.cfg.db.saveBatch(table, batch, {
555
548
  ...opt,
@@ -565,9 +558,6 @@ export class CommonDao {
565
558
  metric: 'saved',
566
559
  ...opt,
567
560
  }),
568
- // just to satisfy and simplify typings
569
- // It's easier to return Transform[], rather than (Transform | Writable)[]
570
- writableVoid(),
571
561
  ];
572
562
  }
573
563
  // DELETE
@@ -600,26 +590,24 @@ export class CommonDao {
600
590
  let deleted = 0;
601
591
  if (opt.chunkSize) {
602
592
  const { chunkSize, chunkConcurrency = 8 } = opt;
603
- await _pipeline([
604
- this.cfg.db.streamQuery(q.select(['id']), opt).map(r => r.id),
605
- transformChunk({ chunkSize }),
606
- transformMap(async (ids) => {
607
- await this.cfg.db.deleteByIds(q.table, ids, opt);
608
- deleted += ids.length;
609
- }, {
610
- predicate: _passthroughPredicate,
611
- concurrency: chunkConcurrency,
612
- errorMode: opt.errorMode || ErrorMode.THROW_IMMEDIATELY,
613
- }),
593
+ await Pipeline.from(this.cfg.db.streamQuery(q.select(['id']), opt).map(r => r.id))
594
+ .chunk(chunkSize)
595
+ .map(async (ids) => {
596
+ await this.cfg.db.deleteByIds(q.table, ids, opt);
597
+ deleted += ids.length;
598
+ }, {
599
+ predicate: _passthroughPredicate,
600
+ concurrency: chunkConcurrency,
601
+ errorMode: opt.errorMode || ErrorMode.THROW_IMMEDIATELY,
602
+ })
614
603
  // LogProgress should be AFTER the mapper, to be able to report correct stats
615
- transformLogProgress({
616
- metric: q.table,
617
- logEvery: 2, // 500 * 2 === 1000
618
- chunkSize,
619
- ...opt,
620
- }),
621
- writableVoid(),
622
- ]);
604
+ .logProgress({
605
+ metric: q.table,
606
+ logEvery: 2, // 500 * 2 === 1000
607
+ chunkSize,
608
+ ...opt,
609
+ })
610
+ .run();
623
611
  }
624
612
  else {
625
613
  deleted = await this.cfg.db.deleteByQuery(q, opt);
package/dist/pipeline/dbPipelineBackup.js CHANGED
@@ -4,8 +4,8 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
4
4
  import { _passthroughMapper } from '@naturalcycles/js-lib/types';
5
5
  import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
6
6
  import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
7
- import { createWriteStreamAsNDJSON, transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
8
- import { _pipeline, NDJsonStats, transformLogProgress, transformMap, transformTap, } from '@naturalcycles/nodejs-lib/stream';
7
+ import { Pipeline, } from '@naturalcycles/nodejs-lib/stream';
8
+ import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
9
9
  import { DBQuery } from '../query/dbQuery.js';
10
10
  /**
11
11
  * Pipeline from input stream(s) to a NDJSON file (optionally gzipped).
@@ -56,24 +56,20 @@ export async function dbPipelineBackup(opt) {
56
56
  await fs2.writeJsonAsync(schemaFilePath, schema, { spaces: 2 });
57
57
  console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`);
58
58
  }
59
- await _pipeline([
60
- db.streamQuery(q),
61
- transformLogProgress({
62
- ...opt,
63
- logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
64
- metric: table,
65
- }),
66
- transformMap(mapperPerTable[table] || _passthroughMapper, {
67
- errorMode,
68
- ...transformMapOptions,
69
- metric: table,
70
- }),
71
- transformFlatten(),
72
- transformTap(() => {
73
- rows++;
74
- }),
75
- ...createWriteStreamAsNDJSON(filePath),
76
- ]);
59
+ await Pipeline.from(db.streamQuery(q))
60
+ .logProgress({
61
+ ...opt,
62
+ logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
63
+ metric: table,
64
+ })
65
+ .map(mapperPerTable[table] || _passthroughMapper, {
66
+ errorMode,
67
+ ...transformMapOptions,
68
+ metric: table,
69
+ })
70
+ .flattenIfNeeded()
71
+ .tap(() => rows++)
72
+ .toNDJsonFile(filePath);
77
73
  const { size: sizeBytes } = await fs2.statAsync(filePath);
78
74
  const stats = NDJsonStats.create({
79
75
  tookMillis: Date.now() - started,
package/dist/pipeline/dbPipelineCopy.js CHANGED
@@ -3,8 +3,8 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
3
3
  import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
4
4
  import { _passthroughMapper } from '@naturalcycles/js-lib/types';
5
5
  import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
6
- import { transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
7
- import { _pipeline, NDJsonStats, transformChunk, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
6
+ import { Pipeline, } from '@naturalcycles/nodejs-lib/stream';
7
+ import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
8
8
  import { DBQuery } from '../query/dbQuery.js';
9
9
  /**
10
10
  * Pipeline from input stream(s) to CommonDB .saveBatch().
@@ -30,25 +30,23 @@ export async function dbPipelineCopy(opt) {
30
30
  const stream = dbInput.streamQuery(q);
31
31
  const started = Date.now();
32
32
  let rows = 0;
33
- await _pipeline([
34
- stream,
35
- transformLogProgress({
36
- logEvery: 1000,
37
- ...opt,
38
- metric: table,
39
- }),
40
- transformMap(mapper, {
41
- errorMode,
42
- ...transformMapOptions,
43
- metric: table,
44
- }),
45
- transformFlatten(),
46
- transformTap(() => rows++),
47
- transformChunk({ chunkSize }),
48
- writableForEach(async (dbms) => {
49
- await dbOutput.saveBatch(table, dbms, saveOptions);
50
- }),
51
- ]);
33
+ await Pipeline.from(stream)
34
+ .logProgress({
35
+ logEvery: 1000,
36
+ ...opt,
37
+ metric: table,
38
+ })
39
+ .map(mapper, {
40
+ errorMode,
41
+ ...transformMapOptions,
42
+ metric: table,
43
+ })
44
+ .flattenIfNeeded()
45
+ .tap(() => rows++)
46
+ .chunk(chunkSize)
47
+ .forEach(async (dbms) => {
48
+ await dbOutput.saveBatch(table, dbms, saveOptions);
49
+ });
52
50
  const stats = NDJsonStats.create({
53
51
  tookMillis: Date.now() - started,
54
52
  rows,
package/dist/pipeline/dbPipelineRestore.js CHANGED
@@ -6,8 +6,8 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
6
6
  import { _passthroughMapper } from '@naturalcycles/js-lib/types';
7
7
  import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors';
8
8
  import { fs2 } from '@naturalcycles/nodejs-lib/fs2';
9
- import { createReadStreamAsNDJSON, transformFlatten, } from '@naturalcycles/nodejs-lib/stream';
10
- import { _pipeline, NDJsonStats, transformChunk, transformFilterSync, transformLogProgress, transformMap, transformTap, writableForEach, } from '@naturalcycles/nodejs-lib/stream';
9
+ import { Pipeline, } from '@naturalcycles/nodejs-lib/stream';
10
+ import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream';
11
11
  /**
12
12
  * Pipeline from NDJSON files in a folder (optionally gzipped) to CommonDB.
13
13
  * Allows to define a mapper and a predicate to map/filter objects between input and output.
@@ -67,28 +67,25 @@ export async function dbPipelineRestore(opt) {
67
67
  let rows = 0;
68
68
  const sizeBytes = sizeByTable[table];
69
69
  console.log(`<< ${grey(filePath)} ${dimWhite(_hb(sizeBytes))} started...`);
70
- await _pipeline([
71
- createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
72
- transformTap(() => rows++),
73
- transformLogProgress({
74
- logEvery: 1000,
75
- ...opt,
76
- metric: table,
77
- }),
78
- ...(sinceUpdated
79
- ? [transformFilterSync(r => r.updated >= sinceUpdated)]
80
- : []),
81
- transformMap(mapperPerTable[table] || _passthroughMapper, {
82
- errorMode,
83
- ...transformMapOptions,
84
- metric: table,
85
- }),
86
- transformFlatten(),
87
- transformChunk({ chunkSize }),
88
- writableForEach(async (dbms) => {
89
- await db.saveBatch(table, dbms, saveOptions);
90
- }),
91
- ]);
70
+ await Pipeline.fromNDJsonFile(filePath)
71
+ .limitSource(limit)
72
+ .tap(() => rows++)
73
+ .logProgress({
74
+ logEvery: 1000,
75
+ ...opt,
76
+ metric: table,
77
+ })
78
+ .filterSync(r => !sinceUpdated || r.updated >= sinceUpdated)
79
+ .map(mapperPerTable[table] || _passthroughMapper, {
80
+ errorMode,
81
+ ...transformMapOptions,
82
+ metric: table,
83
+ })
84
+ .flattenIfNeeded()
85
+ .chunk(chunkSize)
86
+ .forEach(async (dbms) => {
87
+ await db.saveBatch(table, dbms, saveOptions);
88
+ });
92
89
  const stats = NDJsonStats.create({
93
90
  tookMillis: Date.now() - started,
94
91
  rows,
package/dist/query/dbQuery.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import type { AsyncIndexedMapper, BaseDBEntity, ObjectWithId } from '@naturalcycles/js-lib/types';
2
- import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
2
+ import { Pipeline, type ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
3
3
  import type { CommonDao } from '../commondao/common.dao.js';
4
4
  import type { CommonDaoOptions, CommonDaoReadOptions, CommonDaoStreamDeleteOptions, CommonDaoStreamForEachOptions, CommonDaoStreamOptions } from '../commondao/common.dao.model.js';
5
5
  import type { RunQueryResult } from '../db.model.js';
@@ -104,8 +104,11 @@ export declare class RunnableDBQuery<BM extends BaseDBEntity, DBM extends BaseDB
104
104
  streamQueryAsDBMForEach(mapper: AsyncIndexedMapper<DBM, void>, opt?: CommonDaoStreamForEachOptions<DBM>): Promise<void>;
105
105
  streamQuery(opt?: CommonDaoStreamOptions<BM>): ReadableTyped<BM>;
106
106
  streamQueryAsDBM(opt?: CommonDaoStreamOptions<DBM>): ReadableTyped<DBM>;
107
+ pipeline(opt?: CommonDaoStreamOptions<BM>): Pipeline<BM>;
108
+ pipelineAsDBM(opt?: CommonDaoStreamOptions<DBM>): Pipeline<DBM>;
107
109
  queryIds(opt?: CommonDaoReadOptions): Promise<ID[]>;
108
110
  streamQueryIds(opt?: CommonDaoStreamOptions<ID>): ReadableTyped<ID>;
111
+ pipelineIds(opt?: CommonDaoStreamOptions<ID>): Pipeline<ID>;
109
112
  streamQueryIdsForEach(mapper: AsyncIndexedMapper<ID, void>, opt?: CommonDaoStreamForEachOptions<ID>): Promise<void>;
110
113
  deleteByQuery(opt?: CommonDaoStreamDeleteOptions<DBM>): Promise<number>;
111
114
  }
package/dist/query/dbQuery.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { _truncate } from '@naturalcycles/js-lib/string/string.util.js';
2
2
  import { _objectAssign } from '@naturalcycles/js-lib/types';
3
+ import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
3
4
  export const dbQueryFilterOperatorValues = [
4
5
  '<',
5
6
  '<=',
@@ -187,12 +188,21 @@ export class RunnableDBQuery extends DBQuery {
187
188
  streamQueryAsDBM(opt) {
188
189
  return this.dao.streamQueryAsDBM(this, opt);
189
190
  }
191
+ pipeline(opt) {
192
+ return Pipeline.from(this.dao.streamQuery(this, opt));
193
+ }
194
+ pipelineAsDBM(opt) {
195
+ return Pipeline.from(this.dao.streamQueryAsDBM(this, opt));
196
+ }
190
197
  async queryIds(opt) {
191
198
  return await this.dao.queryIds(this, opt);
192
199
  }
193
200
  streamQueryIds(opt) {
194
201
  return this.dao.streamQueryIds(this, opt);
195
202
  }
203
+ pipelineIds(opt) {
204
+ return Pipeline.from(this.dao.streamQueryIds(this, opt));
205
+ }
196
206
  async streamQueryIdsForEach(mapper, opt) {
197
207
  await this.dao.streamQueryIdsForEach(this, mapper, opt);
198
208
  }
package/dist/testing/commonDaoTest.js CHANGED
@@ -1,9 +1,8 @@
1
- import { Readable } from 'node:stream';
2
1
  import { _sortBy } from '@naturalcycles/js-lib/array/array.util.js';
3
2
  import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
4
3
  import { _deepCopy, _filterObject, _omit, _pick } from '@naturalcycles/js-lib/object';
5
4
  import { getJoiValidationFunction } from '@naturalcycles/nodejs-lib/joi';
6
- import { _pipeline } from '@naturalcycles/nodejs-lib/stream';
5
+ import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
7
6
  import { CommonDao } from '../commondao/common.dao.js';
8
7
  import { DBQuery } from '../query/dbQuery.js';
9
8
  import { createTestItemBM, createTestItemsBM, TEST_TABLE, testItemBMJsonSchema, testItemBMSchema, } from './test.model.js';
@@ -203,7 +202,7 @@ export async function runCommonDaoTest(db, quirks = {}) {
203
202
  test('streamSaveTransform', async () => {
204
203
  const items2 = createTestItemsBM(2).map(i => ({ ...i, id: i.id + '_str' }));
205
204
  const ids = items2.map(i => i.id);
206
- await _pipeline([Readable.from(items2), ...dao.streamSaveTransform()]);
205
+ await Pipeline.fromArray(items2).transformMany(dao.streamSaveTransforms()).run();
207
206
  const items2Loaded = await dao.getByIds(ids);
208
207
  expectMatch(items2, items2Loaded, quirks);
209
208
  // cleanup
package/package.json CHANGED
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "@naturalcycles/db-lib",
3
3
  "type": "module",
4
- "version": "10.23.0",
4
+ "version": "10.25.0",
5
5
  "dependencies": {
6
6
  "@naturalcycles/js-lib": "^15",
7
7
  "@naturalcycles/nodejs-lib": "^15"
8
8
  },
9
9
  "devDependencies": {
10
- "@naturalcycles/dev-lib": "18.4.2"
10
+ "@naturalcycles/dev-lib": "19.36.0"
11
11
  },
12
12
  "files": [
13
13
  "dist",
package/src/adapter/file/localFile.persistence.plugin.ts CHANGED
@@ -1,12 +1,7 @@
1
- import { Readable } from 'node:stream'
2
1
  import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
3
2
  import type { ObjectWithId } from '@naturalcycles/js-lib/types'
4
3
  import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
5
- import {
6
- _pipeline,
7
- createReadStreamAsNDJSON,
8
- createWriteStreamAsNDJSON,
9
- } from '@naturalcycles/nodejs-lib/stream'
4
+ import { createReadStreamAsNDJSON, Pipeline } from '@naturalcycles/nodejs-lib/stream'
10
5
  import type { DBSaveBatchOperation } from '../../db.model.js'
11
6
  import type { FileDBPersistencePlugin } from './file.db.model.js'
12
7
 
@@ -63,6 +58,6 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
63
58
  const ext = `ndjson${this.cfg.gzip ? '.gz' : ''}`
64
59
  const filePath = `${this.cfg.storagePath}/${table}.${ext}`
65
60
 
66
- await _pipeline([Readable.from(rows), ...createWriteStreamAsNDJSON(filePath)])
61
+ await Pipeline.fromArray(rows).toNDJsonFile(filePath)
67
62
  }
68
63
  }
package/src/commondao/common.dao.ts CHANGED
@@ -23,17 +23,16 @@ import {
23
23
  import { _passthroughPredicate, _typeCast } from '@naturalcycles/js-lib/types'
24
24
  import { stringId } from '@naturalcycles/nodejs-lib'
25
25
  import {
26
+ Pipeline,
26
27
  type ReadableTyped,
27
28
  transformFlatten,
28
29
  transformMapSync,
29
30
  } from '@naturalcycles/nodejs-lib/stream'
30
31
  import {
31
- _pipeline,
32
32
  transformChunk,
33
33
  transformLogProgress,
34
34
  transformMap,
35
35
  transformNoOp,
36
- writableVoid,
37
36
  } from '@naturalcycles/nodejs-lib/stream'
38
37
  import { DBLibError } from '../cnst.js'
39
38
  import type {
@@ -253,9 +252,8 @@ export class CommonDao<
253
252
 
254
253
  const isPartialQuery = !!q._selectedFieldNames
255
254
 
256
- await _pipeline([
257
- this.cfg.db.streamQuery<DBM>(q, opt),
258
- transformMap<DBM, BM>(
255
+ await Pipeline.from(this.cfg.db.streamQuery<DBM>(q, opt))
256
+ .map(
259
257
  async dbm => {
260
258
  if (isPartialQuery) return dbm as any
261
259
  return await this.dbmToBM(dbm, opt)
@@ -263,18 +261,17 @@ export class CommonDao<
263
261
  {
264
262
  errorMode: opt.errorMode,
265
263
  },
266
- ),
267
- transformMap<BM, void>(mapper, {
264
+ )
265
+ .map(mapper, {
268
266
  ...opt,
269
267
  predicate: _passthroughPredicate, // to be able to logProgress
270
- }),
268
+ })
271
269
  // LogProgress should be AFTER the mapper, to be able to report correct stats
272
- transformLogProgress({
270
+ .logProgress({
273
271
  metric: q.table,
274
272
  ...opt,
275
- }),
276
- writableVoid(),
277
- ])
273
+ })
274
+ .run()
278
275
  }
279
276
 
280
277
  async streamQueryAsDBMForEach(
@@ -289,9 +286,8 @@ export class CommonDao<
289
286
 
290
287
  const isPartialQuery = !!q._selectedFieldNames
291
288
 
292
- await _pipeline([
293
- this.cfg.db.streamQuery<any>(q, opt),
294
- transformMapSync<any, DBM>(
289
+ await Pipeline.from(this.cfg.db.streamQuery<any>(q, opt))
290
+ .mapSync(
295
291
  dbm => {
296
292
  if (isPartialQuery) return dbm
297
293
  return this.anyToDBM(dbm, opt)
@@ -299,18 +295,16 @@ export class CommonDao<
299
295
  {
300
296
  errorMode: opt.errorMode,
301
297
  },
302
- ),
303
- transformMap<DBM, void>(mapper, {
298
+ )
299
+ .map(mapper, {
304
300
  ...opt,
305
301
  predicate: _passthroughPredicate, // to be able to logProgress
306
- }),
307
- // LogProgress should be AFTER the mapper, to be able to report correct stats
308
- transformLogProgress({
302
+ })
303
+ .logProgress({
309
304
  metric: q.table,
310
305
  ...opt,
311
- }),
312
- writableVoid(),
313
- ])
306
+ })
307
+ .run()
314
308
  }
315
309
 
316
310
  /**
@@ -434,19 +428,17 @@ export class CommonDao<
434
428
  q.table = opt.table || q.table
435
429
  opt.errorMode ||= ErrorMode.SUPPRESS
436
430
 
437
- await _pipeline([
438
- this.cfg.db.streamQuery<DBM>(q.select(['id']), opt).map(r => r.id),
439
- transformMap<ID, void>(mapper, {
431
+ await Pipeline.from(this.cfg.db.streamQuery<DBM>(q.select(['id']), opt).map(r => r.id as ID))
432
+ .map(mapper, {
440
433
  ...opt,
441
434
  predicate: _passthroughPredicate,
442
- }),
435
+ })
443
436
  // LogProgress should be AFTER the mapper, to be able to report correct stats
444
- transformLogProgress({
437
+ .logProgress({
445
438
  metric: q.table,
446
439
  ...opt,
447
- }),
448
- writableVoid(),
449
- ])
440
+ })
441
+ .run()
450
442
  }
451
443
 
452
444
  /**
@@ -714,7 +706,7 @@ export class CommonDao<
714
706
  * and then executing db.saveBatch(chunk) with the concurrency
715
707
  * of opt.chunkConcurrency (which defaults to 32).
716
708
  */
717
- streamSaveTransform(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
709
+ streamSaveTransforms(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
718
710
  this.requireWriteAccess()
719
711
 
720
712
  const table = opt.table || this.cfg.table
@@ -742,7 +734,7 @@ export class CommonDao<
742
734
  errorMode,
743
735
  },
744
736
  ),
745
- transformChunk<DBM>({ chunkSize }),
737
+ transformChunk<DBM>(chunkSize),
746
738
  transformMap<DBM[], DBM[]>(
747
739
  async batch => {
748
740
  await this.cfg.db.saveBatch(table, batch, {
@@ -761,9 +753,6 @@ export class CommonDao<
761
753
  metric: 'saved',
762
754
  ...opt,
763
755
  }),
764
- // just to satisfy and simplify typings
765
- // It's easier to return Transform[], rather than (Transform | Writable)[]
766
- writableVoid() as Transform,
767
756
  ]
768
757
  }
769
758
 
@@ -802,10 +791,9 @@ export class CommonDao<
802
791
  if (opt.chunkSize) {
803
792
  const { chunkSize, chunkConcurrency = 8 } = opt
804
793
 
805
- await _pipeline([
806
- this.cfg.db.streamQuery<DBM>(q.select(['id']), opt).map(r => r.id),
807
- transformChunk<string>({ chunkSize }),
808
- transformMap<string[], void>(
794
+ await Pipeline.from(this.cfg.db.streamQuery<DBM>(q.select(['id']), opt).map(r => r.id))
795
+ .chunk(chunkSize)
796
+ .map(
809
797
  async ids => {
810
798
  await this.cfg.db.deleteByIds(q.table, ids, opt)
811
799
  deleted += ids.length
@@ -815,16 +803,15 @@ export class CommonDao<
815
803
  concurrency: chunkConcurrency,
816
804
  errorMode: opt.errorMode || ErrorMode.THROW_IMMEDIATELY,
817
805
  },
818
- ),
806
+ )
819
807
  // LogProgress should be AFTER the mapper, to be able to report correct stats
820
- transformLogProgress({
808
+ .logProgress({
821
809
  metric: q.table,
822
810
  logEvery: 2, // 500 * 2 === 1000
823
811
  chunkSize,
824
812
  ...opt,
825
- }),
826
- writableVoid(),
827
- ])
813
+ })
814
+ .run()
828
815
  } else {
829
816
  deleted = await this.cfg.db.deleteByQuery(q, opt)
830
817
  }
package/src/pipeline/dbPipelineBackup.ts CHANGED
@@ -6,18 +6,11 @@ import { _passthroughMapper } from '@naturalcycles/js-lib/types'
6
6
  import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
7
7
  import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
8
8
  import {
9
- createWriteStreamAsNDJSON,
10
- transformFlatten,
9
+ Pipeline,
11
10
  type TransformLogProgressOptions,
12
11
  type TransformMapOptions,
13
12
  } from '@naturalcycles/nodejs-lib/stream'
14
- import {
15
- _pipeline,
16
- NDJsonStats,
17
- transformLogProgress,
18
- transformMap,
19
- transformTap,
20
- } from '@naturalcycles/nodejs-lib/stream'
13
+ import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream'
21
14
  import type { CommonDB } from '../commondb/common.db.js'
22
15
  import { DBQuery } from '../query/dbQuery.js'
23
16
 
@@ -212,24 +205,20 @@ export async function dbPipelineBackup(opt: DBPipelineBackupOptions): Promise<ND
212
205
  console.log(`>> ${grey(schemaFilePath)} saved (generated from DB)`)
213
206
  }
214
207
 
215
- await _pipeline([
216
- db.streamQuery(q),
217
- transformLogProgress({
208
+ await Pipeline.from(db.streamQuery(q))
209
+ .logProgress({
218
210
  ...opt,
219
211
  logEvery: logEveryPerTable[table] ?? opt.logEvery ?? 1000,
220
212
  metric: table,
221
- }),
222
- transformMap(mapperPerTable[table] || _passthroughMapper, {
213
+ })
214
+ .map(mapperPerTable[table] || _passthroughMapper, {
223
215
  errorMode,
224
216
  ...transformMapOptions,
225
217
  metric: table,
226
- }),
227
- transformFlatten(),
228
- transformTap(() => {
229
- rows++
230
- }),
231
- ...createWriteStreamAsNDJSON(filePath),
232
- ])
218
+ })
219
+ .flattenIfNeeded()
220
+ .tap(() => rows++)
221
+ .toNDJsonFile(filePath)
233
222
 
234
223
  const { size: sizeBytes } = await fs2.statAsync(filePath)
235
224
 
package/src/pipeline/dbPipelineCopy.ts CHANGED
@@ -5,19 +5,11 @@ import type { AsyncMapper, BaseDBEntity, UnixTimestamp } from '@naturalcycles/js
5
5
  import { _passthroughMapper } from '@naturalcycles/js-lib/types'
6
6
  import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
7
7
  import {
8
- transformFlatten,
8
+ Pipeline,
9
9
  type TransformLogProgressOptions,
10
10
  type TransformMapOptions,
11
11
  } from '@naturalcycles/nodejs-lib/stream'
12
- import {
13
- _pipeline,
14
- NDJsonStats,
15
- transformChunk,
16
- transformLogProgress,
17
- transformMap,
18
- transformTap,
19
- writableForEach,
20
- } from '@naturalcycles/nodejs-lib/stream'
12
+ import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream'
21
13
  import type { CommonDB } from '../commondb/common.db.js'
22
14
  import type { CommonDBSaveOptions } from '../db.model.js'
23
15
  import { DBQuery } from '../query/dbQuery.js'
@@ -136,25 +128,23 @@ export async function dbPipelineCopy(opt: DBPipelineCopyOptions): Promise<NDJson
136
128
  const started = Date.now()
137
129
  let rows = 0
138
130
 
139
- await _pipeline([
140
- stream,
141
- transformLogProgress({
131
+ await Pipeline.from(stream)
132
+ .logProgress({
142
133
  logEvery: 1000,
143
134
  ...opt,
144
135
  metric: table,
145
- }),
146
- transformMap(mapper, {
136
+ })
137
+ .map(mapper, {
147
138
  errorMode,
148
139
  ...transformMapOptions,
149
140
  metric: table,
150
- }),
151
- transformFlatten(),
152
- transformTap(() => rows++),
153
- transformChunk({ chunkSize }),
154
- writableForEach(async dbms => {
141
+ })
142
+ .flattenIfNeeded()
143
+ .tap(() => rows++)
144
+ .chunk(chunkSize)
145
+ .forEach(async dbms => {
155
146
  await dbOutput.saveBatch(table, dbms, saveOptions)
156
- }),
157
- ])
147
+ })
158
148
 
159
149
  const stats = NDJsonStats.create({
160
150
  tookMillis: Date.now() - started,
package/src/pipeline/dbPipelineRestore.ts CHANGED
@@ -9,21 +9,11 @@ import { _passthroughMapper } from '@naturalcycles/js-lib/types'
9
9
  import { boldWhite, dimWhite, grey, yellow } from '@naturalcycles/nodejs-lib/colors'
10
10
  import { fs2 } from '@naturalcycles/nodejs-lib/fs2'
11
11
  import {
12
- createReadStreamAsNDJSON,
13
- transformFlatten,
12
+ Pipeline,
14
13
  type TransformLogProgressOptions,
15
14
  type TransformMapOptions,
16
15
  } from '@naturalcycles/nodejs-lib/stream'
17
- import {
18
- _pipeline,
19
- NDJsonStats,
20
- transformChunk,
21
- transformFilterSync,
22
- transformLogProgress,
23
- transformMap,
24
- transformTap,
25
- writableForEach,
26
- } from '@naturalcycles/nodejs-lib/stream'
16
+ import { NDJsonStats } from '@naturalcycles/nodejs-lib/stream'
27
17
  import type { CommonDB } from '../commondb/common.db.js'
28
18
  import type { CommonDBSaveOptions } from '../db.model.js'
29
19
 
@@ -195,28 +185,25 @@ export async function dbPipelineRestore(opt: DBPipelineRestoreOptions): Promise<
195
185
 
196
186
  console.log(`<< ${grey(filePath)} ${dimWhite(_hb(sizeBytes))} started...`)
197
187
 
198
- await _pipeline([
199
- createReadStreamAsNDJSON(filePath).take(limit || Number.POSITIVE_INFINITY),
200
- transformTap(() => rows++),
201
- transformLogProgress({
188
+ await Pipeline.fromNDJsonFile<BaseDBEntity>(filePath)
189
+ .limitSource(limit)
190
+ .tap(() => rows++)
191
+ .logProgress({
202
192
  logEvery: 1000,
203
193
  ...opt,
204
194
  metric: table,
205
- }),
206
- ...(sinceUpdated
207
- ? [transformFilterSync<BaseDBEntity>(r => r.updated >= sinceUpdated)]
208
- : []),
209
- transformMap(mapperPerTable[table] || _passthroughMapper, {
195
+ })
196
+ .filterSync(r => !sinceUpdated || r.updated >= sinceUpdated)
197
+ .map(mapperPerTable[table] || _passthroughMapper, {
210
198
  errorMode,
211
199
  ...transformMapOptions,
212
200
  metric: table,
213
- }),
214
- transformFlatten(),
215
- transformChunk({ chunkSize }),
216
- writableForEach(async dbms => {
201
+ })
202
+ .flattenIfNeeded()
203
+ .chunk(chunkSize)
204
+ .forEach(async dbms => {
217
205
  await db.saveBatch(table, dbms, saveOptions)
218
- }),
219
- ])
206
+ })
220
207
 
221
208
  const stats = NDJsonStats.create({
222
209
  tookMillis: Date.now() - started,
package/src/query/dbQuery.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { _truncate } from '@naturalcycles/js-lib/string/string.util.js'
2
2
  import type { AsyncIndexedMapper, BaseDBEntity, ObjectWithId } from '@naturalcycles/js-lib/types'
3
3
  import { _objectAssign } from '@naturalcycles/js-lib/types'
4
- import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
4
+ import { Pipeline, type ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
5
5
  import type { CommonDao } from '../commondao/common.dao.js'
6
6
  import type {
7
7
  CommonDaoOptions,
@@ -298,6 +298,14 @@ export class RunnableDBQuery<
298
298
  return this.dao.streamQueryAsDBM(this, opt)
299
299
  }
300
300
 
301
+ pipeline(opt?: CommonDaoStreamOptions<BM>): Pipeline<BM> {
302
+ return Pipeline.from(this.dao.streamQuery(this, opt))
303
+ }
304
+
305
+ pipelineAsDBM(opt?: CommonDaoStreamOptions<DBM>): Pipeline<DBM> {
306
+ return Pipeline.from(this.dao.streamQueryAsDBM(this, opt))
307
+ }
308
+
301
309
  async queryIds(opt?: CommonDaoReadOptions): Promise<ID[]> {
302
310
  return await this.dao.queryIds(this, opt)
303
311
  }
@@ -306,6 +314,10 @@ export class RunnableDBQuery<
306
314
  return this.dao.streamQueryIds(this, opt)
307
315
  }
308
316
 
317
+ pipelineIds(opt?: CommonDaoStreamOptions<ID>): Pipeline<ID> {
318
+ return Pipeline.from(this.dao.streamQueryIds(this, opt))
319
+ }
320
+
309
321
  async streamQueryIdsForEach(
310
322
  mapper: AsyncIndexedMapper<ID, void>,
311
323
  opt?: CommonDaoStreamForEachOptions<ID>,
package/src/testing/commonDaoTest.ts CHANGED
@@ -1,9 +1,8 @@
1
- import { Readable } from 'node:stream'
2
1
  import { _sortBy } from '@naturalcycles/js-lib/array/array.util.js'
3
2
  import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
4
3
  import { _deepCopy, _filterObject, _omit, _pick } from '@naturalcycles/js-lib/object'
5
4
  import { getJoiValidationFunction } from '@naturalcycles/nodejs-lib/joi'
6
- import { _pipeline } from '@naturalcycles/nodejs-lib/stream'
5
+ import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
7
6
  import { CommonDao } from '../commondao/common.dao.js'
8
7
  import type { CommonDB } from '../commondb/common.db.js'
9
8
  import { DBQuery } from '../query/dbQuery.js'
@@ -284,7 +283,7 @@ export async function runCommonDaoTest(
284
283
  const items2 = createTestItemsBM(2).map(i => ({ ...i, id: i.id + '_str' }))
285
284
  const ids = items2.map(i => i.id)
286
285
 
287
- await _pipeline([Readable.from(items2), ...dao.streamSaveTransform()])
286
+ await Pipeline.fromArray(items2).transformMany(dao.streamSaveTransforms()).run()
288
287
 
289
288
  const items2Loaded = await dao.getByIds(ids)
290
289
  expectMatch(items2, items2Loaded, quirks)