@naturalcycles/datastore-lib 3.29.0 → 3.30.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/DatastoreStreamReadable.js +17 -9
  2. package/package.json +1 -1
  3. package/src/DatastoreStreamReadable.ts +32 -17
package/dist/DatastoreStreamReadable.js CHANGED
@@ -26,24 +26,28 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
26
26
  batchSize: 1000,
27
27
  ...opt,
28
28
  };
29
- if (this.opt.maxWait) {
29
+ this.originalLimit = q.limitVal;
30
+ this.table = q.kinds[0];
31
+ logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`);
32
+ const { maxWait } = this.opt;
33
+ if (maxWait) {
34
+ this.logger.warn(`!! ${this.table} maxWait ${maxWait}`);
30
35
  this.maxWaitInterval = setInterval(() => {
31
- if (Date.now() - this.lastReadTimestamp < this.opt.maxWait * 1000) {
36
+ const millisSinceLastRead = Date.now() - this.lastReadTimestamp;
37
+ if (millisSinceLastRead < maxWait * 1000) {
38
+ this.logger.warn(`!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`);
32
39
  return;
33
40
  }
34
41
  const { running, rowsRetrieved } = this;
35
- this.logger.warn(`maxWait of ${this.opt.maxWait} seconds reached, force-triggering _read`, {
42
+ this.logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
36
43
  running,
37
44
  rowsRetrieved,
38
45
  });
39
46
  // force-trigger _read
40
47
  // regardless of `running` status
41
48
  this._read();
42
- }, (this.opt.maxWait * 1000) / 2);
49
+ }, (maxWait * 1000) / 2);
43
50
  }
44
- this.originalLimit = q.limitVal;
45
- this.table = q.kinds[0];
46
- logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`);
47
51
  }
48
52
  async runNextQuery() {
49
53
  if (this.done)
@@ -85,10 +89,14 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
85
89
  rowsRetrieved: this.rowsRetrieved,
86
90
  }, err);
87
91
  this.emit('error', err);
92
+ clearInterval(this.maxWaitInterval);
88
93
  return;
89
94
  }
90
95
  this.rowsRetrieved += rows.length;
91
- this.logger.log(`got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
96
+ this.logger.log(`${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
97
+ if (!rows.length) {
98
+ this.logger.warn(`${this.table} got 0 rows, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
99
+ }
92
100
  this.endCursor = info.endCursor;
93
101
  this.running = false; // ready to take more _reads
94
102
  this.lastQueryDone = Date.now();
@@ -111,7 +119,7 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
111
119
  void this.runNextQuery();
112
120
  }
113
121
  else {
114
- this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
122
+ this.logger.warn(`${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
115
123
  }
116
124
  }
117
125
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@naturalcycles/datastore-lib",
3
- "version": "3.29.0",
3
+ "version": "3.30.1",
4
4
  "description": "Opinionated library to work with Google Datastore",
5
5
  "scripts": {
6
6
  "prepare": "husky install"
package/src/DatastoreStreamReadable.ts CHANGED
@@ -35,34 +35,39 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
35
35
  ...opt,
36
36
  }
37
37
 
38
- if (this.opt.maxWait) {
38
+ this.originalLimit = q.limitVal
39
+ this.table = q.kinds[0]!
40
+
41
+ logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`)
42
+
43
+ const { maxWait } = this.opt
44
+ if (maxWait) {
45
+ this.logger.warn(`!! ${this.table} maxWait ${maxWait}`)
46
+
39
47
  this.maxWaitInterval = setInterval(
40
48
  () => {
41
- if (Date.now() - this.lastReadTimestamp < this.opt.maxWait! * 1000) {
49
+ const millisSinceLastRead = Date.now() - this.lastReadTimestamp
50
+
51
+ if (millisSinceLastRead < maxWait * 1000) {
52
+ this.logger.warn(
53
+ `!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`,
54
+ )
42
55
  return
43
56
  }
44
57
 
45
58
  const { running, rowsRetrieved } = this
46
- this.logger.warn(
47
- `maxWait of ${this.opt.maxWait} seconds reached, force-triggering _read`,
48
- {
49
- running,
50
- rowsRetrieved,
51
- },
52
- )
59
+ this.logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
60
+ running,
61
+ rowsRetrieved,
62
+ })
53
63
 
54
64
  // force-trigger _read
55
65
  // regardless of `running` status
56
66
  this._read()
57
67
  },
58
- (this.opt.maxWait * 1000) / 2,
68
+ (maxWait * 1000) / 2,
59
69
  )
60
70
  }
61
-
62
- this.originalLimit = q.limitVal
63
- this.table = q.kinds[0]!
64
-
65
- logger.log(`!! using experimentalCursorStream !! ${this.table}, batchSize: ${opt.batchSize}`)
66
71
  }
67
72
 
68
73
  private async runNextQuery(): Promise<void> {
@@ -117,17 +122,25 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
117
122
  err,
118
123
  )
119
124
  this.emit('error', err)
125
+ clearInterval(this.maxWaitInterval)
120
126
  return
121
127
  }
122
128
 
123
129
  this.rowsRetrieved += rows.length
124
130
  this.logger.log(
125
- `got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
131
+ `${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
126
132
  this.totalWait,
127
133
  )}`,
128
134
  info.moreResults,
129
135
  )
130
136
 
137
+ if (!rows.length) {
138
+ this.logger.warn(
139
+ `${this.table} got 0 rows, totalWait: ${_ms(this.totalWait)}`,
140
+ info.moreResults,
141
+ )
142
+ }
143
+
131
144
  this.endCursor = info.endCursor
132
145
  this.running = false // ready to take more _reads
133
146
  this.lastQueryDone = Date.now()
@@ -154,7 +167,9 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
154
167
  if (rssMB <= this.opt.rssLimitMB) {
155
168
  void this.runNextQuery()
156
169
  } else {
157
- this.logger.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`)
170
+ this.logger.warn(
171
+ `${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`,
172
+ )
158
173
  }
159
174
  }
160
175
  }