@naturalcycles/firestore-lib 2.8.1 → 2.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/dist/firestoreStreamReadable.d.ts +2 -1
  2. package/dist/firestoreStreamReadable.js +28 -18
  3. package/package.json +1 -1
  4. package/src/firestoreStreamReadable.ts +33 -21
package/dist/firestoreStreamReadable.d.ts CHANGED
@@ -12,7 +12,8 @@ export declare class FirestoreStreamReadable<T extends ObjectWithId = any> exten
12
12
  private readonly originalLimit;
13
13
  private rowsRetrieved;
14
14
  private endCursor?;
15
- private running;
15
+ private queryIsRunning;
16
+ private paused;
16
17
  private done;
17
18
  private lastQueryDone?;
18
19
  private totalWait;
package/dist/firestoreStreamReadable.js CHANGED
@@ -10,7 +10,8 @@ export class FirestoreStreamReadable extends Readable {
10
10
  originalLimit;
11
11
  rowsRetrieved = 0;
12
12
  endCursor;
13
- running = false;
13
+ queryIsRunning = false;
14
+ paused = false;
14
15
  done = false;
15
16
  lastQueryDone;
16
17
  totalWait = 0;
@@ -48,24 +49,26 @@ export class FirestoreStreamReadable extends Readable {
48
49
  this.logger.warn(`!!! _read was called, but done==true`);
49
50
  return;
50
51
  }
51
- if (!this.running) {
52
+ if (!this.queryIsRunning) {
52
53
  void this.runNextQuery().catch(err => {
53
54
  console.log('error in runNextQuery', err);
54
55
  this.emit('error', err);
55
56
  });
56
57
  }
57
58
  else {
58
- this.logger.log(`_read ${this.count}, wasRunning: true`);
59
+ this.logger.log(`_read ${this.count}, queryIsRunning: true`);
60
+ // todo: check if this can cause a "hang", if no more _reads would come later and we get stuck?
59
61
  }
60
62
  }
61
63
  async runNextQuery() {
62
64
  if (this.done)
63
65
  return;
66
+ const { logger, table } = this;
64
67
  if (this.lastQueryDone) {
65
68
  const now = Date.now();
66
69
  this.totalWait += now - this.lastQueryDone;
67
70
  }
68
- this.running = true;
71
+ this.queryIsRunning = true;
69
72
  let limit = this.opt.batchSize;
70
73
  if (this.originalLimit) {
71
74
  limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved);
@@ -81,17 +84,17 @@ export class FirestoreStreamReadable extends Readable {
81
84
  await pRetry(async () => {
82
85
  qs = await q.get();
83
86
  }, {
84
- name: `FirestoreStreamReadable.query(${this.table})`,
87
+ name: `FirestoreStreamReadable.query(${table})`,
85
88
  maxAttempts: 5,
86
89
  delay: 5000,
87
90
  delayMultiplier: 2,
88
- logger: this.logger,
91
+ logger,
89
92
  timeout: 120_000, // 2 minutes
90
93
  });
91
94
  }
92
95
  catch (err) {
93
96
  console.log(`FirestoreStreamReadable error!\n`, {
94
- table: this.table,
97
+ table,
95
98
  rowsRetrieved: this.rowsRetrieved,
96
99
  }, err);
97
100
  this.emit('error', err);
@@ -108,30 +111,37 @@ export class FirestoreStreamReadable extends Readable {
108
111
  });
109
112
  }
110
113
  this.rowsRetrieved += rows.length;
111
- this.logger.log(`${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
114
+ logger.log(`${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
112
115
  this.endCursor = lastDocId;
113
- this.running = false; // ready to take more _reads
116
+ this.queryIsRunning = false; // ready to take more _reads
114
117
  this.lastQueryDone = Date.now();
115
118
  for (const row of rows) {
116
119
  this.push(row);
117
120
  }
118
121
  if (qs.empty || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
119
- this.logger.warn(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
122
+ logger.warn(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
120
123
  this.push(null);
124
+ this.paused = false;
121
125
  this.done = true;
126
+ return;
122
127
  }
123
- else if (this.opt.singleBatchBuffer) {
128
+ if (this.opt.singleBatchBuffer) {
124
129
  // here we don't start next query until we're asked (via next _read call)
125
130
  // so, let's do nothing
131
+ return;
126
132
  }
127
- else {
128
- const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
129
- if (rssMB <= this.opt.rssLimitMB) {
130
- void this.runNextQuery();
131
- }
132
- else {
133
- this.logger.warn(`${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
133
+ const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
134
+ const { rssLimitMB } = this.opt;
135
+ if (rssMB <= rssLimitMB) {
136
+ if (this.paused) {
137
+ logger.warn(`${table} rssLimitMB is below ${rssMB} < ${rssLimitMB}, unpausing stream`);
138
+ this.paused = false;
134
139
  }
140
+ void this.runNextQuery();
141
+ }
142
+ else if (!this.paused) {
143
+ logger.warn(`${table} rssLimitMB reached ${rssMB} > ${rssLimitMB}, pausing stream`);
144
+ this.paused = true;
135
145
  }
136
146
  }
137
147
  }
package/package.json CHANGED
@@ -38,7 +38,7 @@
38
38
  "engines": {
39
39
  "node": ">=22.12.0"
40
40
  },
41
- "version": "2.8.1",
41
+ "version": "2.9.0",
42
42
  "description": "Firestore implementation of CommonDB interface",
43
43
  "author": "Natural Cycles Team",
44
44
  "license": "MIT",
package/src/firestoreStreamReadable.ts CHANGED
@@ -17,7 +17,8 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
17
17
  private readonly originalLimit: number
18
18
  private rowsRetrieved = 0
19
19
  private endCursor?: string
20
- private running = false
20
+ private queryIsRunning = false
21
+ private paused = false
21
22
  private done = false
22
23
  private lastQueryDone?: number
23
24
  private totalWait = 0
@@ -70,25 +71,27 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
70
71
  return
71
72
  }
72
73
 
73
- if (!this.running) {
74
+ if (!this.queryIsRunning) {
74
75
  void this.runNextQuery().catch(err => {
75
76
  console.log('error in runNextQuery', err)
76
77
  this.emit('error', err)
77
78
  })
78
79
  } else {
79
- this.logger.log(`_read ${this.count}, wasRunning: true`)
80
+ this.logger.log(`_read ${this.count}, queryIsRunning: true`)
81
+ // todo: check if this can cause a "hang", if no more _reads would come later and we get stuck?
80
82
  }
81
83
  }
82
84
 
83
85
  private async runNextQuery(): Promise<void> {
84
86
  if (this.done) return
87
+ const { logger, table } = this
85
88
 
86
89
  if (this.lastQueryDone) {
87
90
  const now = Date.now()
88
91
  this.totalWait += now - this.lastQueryDone
89
92
  }
90
93
 
91
- this.running = true
94
+ this.queryIsRunning = true
92
95
 
93
96
  let limit = this.opt.batchSize
94
97
 
@@ -111,11 +114,11 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
111
114
  qs = await q.get()
112
115
  },
113
116
  {
114
- name: `FirestoreStreamReadable.query(${this.table})`,
117
+ name: `FirestoreStreamReadable.query(${table})`,
115
118
  maxAttempts: 5,
116
119
  delay: 5000,
117
120
  delayMultiplier: 2,
118
- logger: this.logger,
121
+ logger,
119
122
  timeout: 120_000, // 2 minutes
120
123
  },
121
124
  )
@@ -123,7 +126,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
123
126
  console.log(
124
127
  `FirestoreStreamReadable error!\n`,
125
128
  {
126
- table: this.table,
129
+ table,
127
130
  rowsRetrieved: this.rowsRetrieved,
128
131
  },
129
132
  err,
@@ -145,14 +148,14 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
145
148
  }
146
149
 
147
150
  this.rowsRetrieved += rows.length
148
- this.logger.log(
149
- `${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
151
+ logger.log(
152
+ `${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
150
153
  this.totalWait,
151
154
  )}`,
152
155
  )
153
156
 
154
157
  this.endCursor = lastDocId
155
- this.running = false // ready to take more _reads
158
+ this.queryIsRunning = false // ready to take more _reads
156
159
  this.lastQueryDone = Date.now()
157
160
 
158
161
  for (const row of rows) {
@@ -160,24 +163,33 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
160
163
  }
161
164
 
162
165
  if (qs!.empty || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
163
- this.logger.warn(
166
+ logger.warn(
164
167
  `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
165
168
  )
166
169
  this.push(null)
170
+ this.paused = false
167
171
  this.done = true
168
- } else if (this.opt.singleBatchBuffer) {
172
+ return
173
+ }
174
+
175
+ if (this.opt.singleBatchBuffer) {
169
176
  // here we don't start next query until we're asked (via next _read call)
170
177
  // so, let's do nothing
171
- } else {
172
- const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
173
-
174
- if (rssMB <= this.opt.rssLimitMB) {
175
- void this.runNextQuery()
176
- } else {
177
- this.logger.warn(
178
- `${this.table} rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`,
179
- )
178
+ return
179
+ }
180
+
181
+ const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
182
+ const { rssLimitMB } = this.opt
183
+
184
+ if (rssMB <= rssLimitMB) {
185
+ if (this.paused) {
186
+ logger.warn(`${table} rssLimitMB is below ${rssMB} < ${rssLimitMB}, unpausing stream`)
187
+ this.paused = false
180
188
  }
189
+ void this.runNextQuery()
190
+ } else if (!this.paused) {
191
+ logger.warn(`${table} rssLimitMB reached ${rssMB} > ${rssLimitMB}, pausing stream`)
192
+ this.paused = true
181
193
  }
182
194
  }
183
195
  }