@naturalcycles/datastore-lib 3.22.0 → 3.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -54,6 +54,10 @@ class DatastoreStreamReadable extends stream_1.Readable {
|
|
|
54
54
|
this.push(null);
|
|
55
55
|
this.done = true;
|
|
56
56
|
}
|
|
57
|
+
else if (this.opt.singleBatchBuffer) {
|
|
58
|
+
// here we don't start next query until we're asked (via next _read call)
|
|
59
|
+
// do, let's do nothing
|
|
60
|
+
}
|
|
57
61
|
else if (this.opt.rssLimitMB) {
|
|
58
62
|
const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
|
|
59
63
|
if (rssMB <= this.opt.rssLimitMB) {
|
|
@@ -60,7 +60,8 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
|
|
|
60
60
|
*/
|
|
61
61
|
experimentalCursorStream?: boolean;
|
|
62
62
|
/**
|
|
63
|
-
* Applicable to `experimentalCursorStream
|
|
63
|
+
* Applicable to `experimentalCursorStream`.
|
|
64
|
+
* Defines the size (limit) of each individual query.
|
|
64
65
|
*
|
|
65
66
|
* @default 1000
|
|
66
67
|
*/
|
|
@@ -78,6 +79,16 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
|
|
|
78
79
|
* @default 1000
|
|
79
80
|
*/
|
|
80
81
|
rssLimitMB?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Applicable to `experimentalCursorStream`
|
|
84
|
+
* Default false.
|
|
85
|
+
* If true, stream will pause until consumer requests more data (via _read).
|
|
86
|
+
* It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
|
|
87
|
+
* There will be gaps in time between "last query loaded" and "next query requested".
|
|
88
|
+
* This mode is useful e.g for DB migrations, where you want to avoid "stale data".
|
|
89
|
+
* So, it minimizes the time between "item loaded" and "item saved" during DB migration.
|
|
90
|
+
*/
|
|
91
|
+
singleBatchBuffer?: boolean;
|
|
81
92
|
/**
|
|
82
93
|
* Set to `true` to log additional debug info, when using experimentalCursorStream.
|
|
83
94
|
*
|
package/package.json
CHANGED
|
@@ -79,6 +79,9 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
|
|
|
79
79
|
)
|
|
80
80
|
this.push(null)
|
|
81
81
|
this.done = true
|
|
82
|
+
} else if (this.opt.singleBatchBuffer) {
|
|
83
|
+
// here we don't start next query until we're asked (via next _read call)
|
|
84
|
+
// do, let's do nothing
|
|
82
85
|
} else if (this.opt.rssLimitMB) {
|
|
83
86
|
const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
|
|
84
87
|
|
package/src/datastore.model.ts
CHANGED
|
@@ -71,7 +71,8 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
|
|
|
71
71
|
experimentalCursorStream?: boolean
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
|
-
* Applicable to `experimentalCursorStream
|
|
74
|
+
* Applicable to `experimentalCursorStream`.
|
|
75
|
+
* Defines the size (limit) of each individual query.
|
|
75
76
|
*
|
|
76
77
|
* @default 1000
|
|
77
78
|
*/
|
|
@@ -91,6 +92,17 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
|
|
|
91
92
|
*/
|
|
92
93
|
rssLimitMB?: number
|
|
93
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Applicable to `experimentalCursorStream`
|
|
97
|
+
* Default false.
|
|
98
|
+
* If true, stream will pause until consumer requests more data (via _read).
|
|
99
|
+
* It means it'll run slower, as buffer will be equal to batchSize (1000) at max.
|
|
100
|
+
* There will be gaps in time between "last query loaded" and "next query requested".
|
|
101
|
+
* This mode is useful e.g for DB migrations, where you want to avoid "stale data".
|
|
102
|
+
* So, it minimizes the time between "item loaded" and "item saved" during DB migration.
|
|
103
|
+
*/
|
|
104
|
+
singleBatchBuffer?: boolean
|
|
105
|
+
|
|
94
106
|
/**
|
|
95
107
|
* Set to `true` to log additional debug info, when using experimentalCursorStream.
|
|
96
108
|
*
|