@naturalcycles/db-lib 9.12.1 → 9.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/file/localFile.persistence.plugin.js +1 -1
- package/dist/commondao/common.dao.d.ts +1 -1
- package/dist/commondao/common.dao.js +3 -3
- package/dist/commondao/common.dao.model.d.ts +1 -1
- package/dist/kv/commonKeyValueDao.js +2 -2
- package/package.json +1 -1
- package/src/adapter/file/localFile.persistence.plugin.ts +1 -1
- package/src/commondao/common.dao.model.ts +1 -1
- package/src/commondao/common.dao.ts +3 -3
- package/src/kv/commonKeyValueDao.ts +2 -2
|
@@ -43,7 +43,7 @@ class LocalFilePersistencePlugin {
|
|
|
43
43
|
return rows;
|
|
44
44
|
}
|
|
45
45
|
async saveFiles(ops) {
|
|
46
|
-
await (0, js_lib_1.pMap)(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency:
|
|
46
|
+
await (0, js_lib_1.pMap)(ops, async (op) => await this.saveFile(op.table, op.rows), { concurrency: 32 });
|
|
47
47
|
}
|
|
48
48
|
async saveFile(table, rows) {
|
|
49
49
|
await nodejs_lib_1.fs2.ensureDirAsync(this.cfg.storagePath);
|
|
@@ -114,7 +114,7 @@ export declare class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity
|
|
|
114
114
|
* "Streaming" is implemented by buffering incoming rows into **batches**
|
|
115
115
|
* (of size opt.chunkSize, which defaults to 500),
|
|
116
116
|
* and then executing db.saveBatch(chunk) with the concurrency
|
|
117
|
-
* of opt.chunkConcurrency (which defaults to
|
|
117
|
+
* of opt.chunkConcurrency (which defaults to 32).
|
|
118
118
|
*/
|
|
119
119
|
streamSaveTransform(opt?: CommonDaoStreamSaveOptions<DBM>): Transform[];
|
|
120
120
|
/**
|
|
@@ -699,7 +699,7 @@ class CommonDao {
|
|
|
699
699
|
* "Streaming" is implemented by buffering incoming rows into **batches**
|
|
700
700
|
* (of size opt.chunkSize, which defaults to 500),
|
|
701
701
|
* and then executing db.saveBatch(chunk) with the concurrency
|
|
702
|
-
* of opt.chunkConcurrency (which defaults to
|
|
702
|
+
* of opt.chunkConcurrency (which defaults to 32).
|
|
703
703
|
*/
|
|
704
704
|
streamSaveTransform(opt = {}) {
|
|
705
705
|
this.requireWriteAccess();
|
|
@@ -711,7 +711,7 @@ class CommonDao {
|
|
|
711
711
|
}
|
|
712
712
|
const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes;
|
|
713
713
|
const { beforeSave } = this.cfg.hooks;
|
|
714
|
-
const { chunkSize = 500, chunkConcurrency =
|
|
714
|
+
const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt;
|
|
715
715
|
return [
|
|
716
716
|
(0, nodejs_lib_1.transformMap)(async (bm) => {
|
|
717
717
|
this.assignIdCreatedUpdated(bm, opt); // mutates
|
|
@@ -780,7 +780,7 @@ class CommonDao {
|
|
|
780
780
|
const started = this.logStarted(op, q.table);
|
|
781
781
|
let deleted = 0;
|
|
782
782
|
if (opt.chunkSize) {
|
|
783
|
-
const { chunkSize, chunkConcurrency =
|
|
783
|
+
const { chunkSize, chunkConcurrency = 32 } = opt;
|
|
784
784
|
await (0, nodejs_lib_1._pipeline)([
|
|
785
785
|
this.cfg.db.streamQuery(q.select(['id']), opt),
|
|
786
786
|
(0, nodejs_lib_1.transformMapSimple)(r => r.id, {
|
|
@@ -257,7 +257,7 @@ export interface CommonDaoStreamOptions<IN> extends CommonDaoOptions, TransformL
|
|
|
257
257
|
chunkSize?: number;
|
|
258
258
|
/**
|
|
259
259
|
* When chunkSize is set - this option controls how many chunks to run concurrently.
|
|
260
|
-
* Defaults to
|
|
260
|
+
* Defaults to 32.
|
|
261
261
|
*/
|
|
262
262
|
chunkConcurrency?: number;
|
|
263
263
|
}
|
|
@@ -137,7 +137,7 @@ class CommonKeyValueDao {
|
|
|
137
137
|
return []; // SKIP
|
|
138
138
|
}
|
|
139
139
|
}, {
|
|
140
|
-
concurrency:
|
|
140
|
+
concurrency: 32,
|
|
141
141
|
});
|
|
142
142
|
}
|
|
143
143
|
streamEntries(limit) {
|
|
@@ -154,7 +154,7 @@ class CommonKeyValueDao {
|
|
|
154
154
|
return []; // SKIP
|
|
155
155
|
}
|
|
156
156
|
}, {
|
|
157
|
-
concurrency:
|
|
157
|
+
concurrency: 32,
|
|
158
158
|
});
|
|
159
159
|
}
|
|
160
160
|
}
|
package/package.json
CHANGED
|
@@ -71,7 +71,7 @@ export class LocalFilePersistencePlugin implements FileDBPersistencePlugin {
|
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
async saveFiles(ops: DBSaveBatchOperation<any>[]): Promise<void> {
|
|
74
|
-
await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency:
|
|
74
|
+
await pMap(ops, async op => await this.saveFile(op.table, op.rows), { concurrency: 32 })
|
|
75
75
|
}
|
|
76
76
|
|
|
77
77
|
async saveFile<ROW extends ObjectWithId>(table: string, rows: ROW[]): Promise<void> {
|
|
@@ -321,7 +321,7 @@ export interface CommonDaoStreamOptions<IN>
|
|
|
321
321
|
|
|
322
322
|
/**
|
|
323
323
|
* When chunkSize is set - this option controls how many chunks to run concurrently.
|
|
324
|
-
* Defaults to
|
|
324
|
+
* Defaults to 32.
|
|
325
325
|
*/
|
|
326
326
|
chunkConcurrency?: number
|
|
327
327
|
}
|
|
@@ -920,7 +920,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
|
|
|
920
920
|
* "Streaming" is implemented by buffering incoming rows into **batches**
|
|
921
921
|
* (of size opt.chunkSize, which defaults to 500),
|
|
922
922
|
* and then executing db.saveBatch(chunk) with the concurrency
|
|
923
|
-
* of opt.chunkConcurrency (which defaults to
|
|
923
|
+
* of opt.chunkConcurrency (which defaults to 32).
|
|
924
924
|
*/
|
|
925
925
|
streamSaveTransform(opt: CommonDaoStreamSaveOptions<DBM> = {}): Transform[] {
|
|
926
926
|
this.requireWriteAccess()
|
|
@@ -936,7 +936,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
|
|
|
936
936
|
const excludeFromIndexes = opt.excludeFromIndexes || this.cfg.excludeFromIndexes
|
|
937
937
|
const { beforeSave } = this.cfg.hooks!
|
|
938
938
|
|
|
939
|
-
const { chunkSize = 500, chunkConcurrency =
|
|
939
|
+
const { chunkSize = 500, chunkConcurrency = 32, errorMode } = opt
|
|
940
940
|
|
|
941
941
|
return [
|
|
942
942
|
transformMap<BM, DBM>(
|
|
@@ -1019,7 +1019,7 @@ export class CommonDao<BM extends BaseDBEntity, DBM extends BaseDBEntity = BM> {
|
|
|
1019
1019
|
let deleted = 0
|
|
1020
1020
|
|
|
1021
1021
|
if (opt.chunkSize) {
|
|
1022
|
-
const { chunkSize, chunkConcurrency =
|
|
1022
|
+
const { chunkSize, chunkConcurrency = 32 } = opt
|
|
1023
1023
|
|
|
1024
1024
|
await _pipeline([
|
|
1025
1025
|
this.cfg.db.streamQuery<DBM>(q.select(['id']), opt),
|
|
@@ -224,7 +224,7 @@ export class CommonKeyValueDao<T> {
|
|
|
224
224
|
}
|
|
225
225
|
},
|
|
226
226
|
{
|
|
227
|
-
concurrency:
|
|
227
|
+
concurrency: 32,
|
|
228
228
|
},
|
|
229
229
|
)
|
|
230
230
|
}
|
|
@@ -248,7 +248,7 @@ export class CommonKeyValueDao<T> {
|
|
|
248
248
|
}
|
|
249
249
|
},
|
|
250
250
|
{
|
|
251
|
-
concurrency:
|
|
251
|
+
concurrency: 32,
|
|
252
252
|
},
|
|
253
253
|
)
|
|
254
254
|
}
|