@naturalcycles/firestore-lib 2.11.0 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/firestore.db.d.ts +6 -5
- package/dist/firestore.db.js +4 -3
- package/dist/firestoreShardedReadable.d.ts +2 -3
- package/dist/firestoreShardedReadable.js +16 -14
- package/dist/firestoreStreamReadable.d.ts +2 -3
- package/dist/firestoreStreamReadable.js +13 -11
- package/package.json +1 -1
- package/src/firestore.db.ts +12 -17
- package/src/firestoreShardedReadable.ts +15 -13
- package/src/firestoreStreamReadable.ts +12 -10
package/dist/firestore.db.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Firestore, Query, QuerySnapshot, Transaction } from '@google-cloud/firestore';
|
|
2
2
|
import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions, CommonDBSupport, CommonDBTransactionOptions, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult } from '@naturalcycles/db-lib';
|
|
3
3
|
import { BaseCommonDB } from '@naturalcycles/db-lib';
|
|
4
|
-
import {
|
|
4
|
+
import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log';
|
|
5
5
|
import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types';
|
|
6
6
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
7
7
|
export declare class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
@@ -55,6 +55,7 @@ export interface FirestoreDBCfg {
|
|
|
55
55
|
* Default to `console`
|
|
56
56
|
*/
|
|
57
57
|
logger?: CommonLogger;
|
|
58
|
+
logLevel?: CommonLogLevel;
|
|
58
59
|
}
|
|
59
60
|
export declare class RollbackError extends Error {
|
|
60
61
|
constructor();
|
|
@@ -82,12 +83,12 @@ export interface FirestoreDBStreamOptions extends FirestoreDBReadOptions {
|
|
|
82
83
|
* between the queries.
|
|
83
84
|
*/
|
|
84
85
|
highWaterMark?: PositiveInteger;
|
|
86
|
+
logger?: CommonLogger;
|
|
85
87
|
/**
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
* @default false
|
|
88
|
+
* Defaults to `log`.
|
|
89
|
+
* Set to `debug` to allow for extra debugging, e.g in experimentalCursorStream.
|
|
89
90
|
*/
|
|
90
|
-
|
|
91
|
+
logLevel?: CommonLogLevel;
|
|
91
92
|
}
|
|
92
93
|
export interface FirestoreDBOptions extends CommonDBOptions {
|
|
93
94
|
}
|
package/dist/firestore.db.js
CHANGED
|
@@ -3,7 +3,6 @@ import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
|
|
|
3
3
|
import { _isTruthy } from '@naturalcycles/js-lib';
|
|
4
4
|
import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
|
|
5
5
|
import { _assert } from '@naturalcycles/js-lib/error/assert.js';
|
|
6
|
-
import { commonLoggerMinLevel } from '@naturalcycles/js-lib/log';
|
|
7
6
|
import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/object.util.js';
|
|
8
7
|
import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
|
|
9
8
|
import { _stringMapEntries } from '@naturalcycles/js-lib/types';
|
|
@@ -96,14 +95,16 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
96
95
|
streamQuery(q, opt_) {
|
|
97
96
|
const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table));
|
|
98
97
|
const opt = {
|
|
98
|
+
logger: this.cfg.logger,
|
|
99
|
+
logLevel: this.cfg.logLevel,
|
|
99
100
|
...this.cfg.streamOptions,
|
|
100
101
|
...opt_,
|
|
101
102
|
};
|
|
102
103
|
if (opt.experimentalCursorStream) {
|
|
103
|
-
return new FirestoreStreamReadable(firestoreQuery, q, opt
|
|
104
|
+
return new FirestoreStreamReadable(firestoreQuery, q, opt);
|
|
104
105
|
}
|
|
105
106
|
if (opt.experimentalShardedStream) {
|
|
106
|
-
return new FirestoreShardedReadable(firestoreQuery, q, opt
|
|
107
|
+
return new FirestoreShardedReadable(firestoreQuery, q, opt);
|
|
107
108
|
}
|
|
108
109
|
return firestoreQuery.stream().map(doc => {
|
|
109
110
|
return {
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import { type Query } from '@google-cloud/firestore';
|
|
3
3
|
import type { DBQuery } from '@naturalcycles/db-lib';
|
|
4
|
-
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
5
4
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
6
5
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
7
6
|
import type { FirestoreDBStreamOptions } from './firestore.db.js';
|
|
@@ -11,7 +10,6 @@ import type { FirestoreDBStreamOptions } from './firestore.db.js';
|
|
|
11
10
|
export declare class FirestoreShardedReadable<T extends ObjectWithId = any> extends Readable implements ReadableTyped<T> {
|
|
12
11
|
private readonly q;
|
|
13
12
|
readonly dbQuery: DBQuery<T>;
|
|
14
|
-
private logger;
|
|
15
13
|
private readonly table;
|
|
16
14
|
private readonly originalLimit;
|
|
17
15
|
private rowsRetrieved;
|
|
@@ -27,7 +25,8 @@ export declare class FirestoreShardedReadable<T extends ObjectWithId = any> exte
|
|
|
27
25
|
private lastQueryDoneByShard;
|
|
28
26
|
private totalWait;
|
|
29
27
|
private readonly opt;
|
|
30
|
-
|
|
28
|
+
private logger;
|
|
29
|
+
constructor(q: Query, dbQuery: DBQuery<T>, opt: FirestoreDBStreamOptions);
|
|
31
30
|
/**
|
|
32
31
|
* Counts how many times _read was called.
|
|
33
32
|
* For debugging.
|
|
@@ -2,6 +2,7 @@ import { Readable } from 'node:stream';
|
|
|
2
2
|
import { FieldPath } from '@google-cloud/firestore';
|
|
3
3
|
import { localTime } from '@naturalcycles/js-lib/datetime';
|
|
4
4
|
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
|
|
5
|
+
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
|
|
5
6
|
import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
|
|
6
7
|
import { unescapeDocId } from './firestore.util.js';
|
|
7
8
|
const SHARDS = 16;
|
|
@@ -12,7 +13,6 @@ const SHARD_COLUMN = 'shard16';
|
|
|
12
13
|
export class FirestoreShardedReadable extends Readable {
|
|
13
14
|
q;
|
|
14
15
|
dbQuery;
|
|
15
|
-
logger;
|
|
16
16
|
table;
|
|
17
17
|
originalLimit;
|
|
18
18
|
rowsRetrieved = 0;
|
|
@@ -29,18 +29,20 @@ export class FirestoreShardedReadable extends Readable {
|
|
|
29
29
|
lastQueryDoneByShard = {};
|
|
30
30
|
totalWait = 0;
|
|
31
31
|
opt;
|
|
32
|
-
|
|
32
|
+
logger;
|
|
33
|
+
constructor(q, dbQuery, opt) {
|
|
33
34
|
super({ objectMode: true });
|
|
34
35
|
this.q = q;
|
|
35
36
|
this.dbQuery = dbQuery;
|
|
36
|
-
this.logger = logger;
|
|
37
37
|
this.opt = {
|
|
38
38
|
batchSize: 3000,
|
|
39
39
|
...opt,
|
|
40
40
|
};
|
|
41
41
|
this.originalLimit = dbQuery._limitValue;
|
|
42
42
|
this.table = dbQuery.table;
|
|
43
|
-
logger
|
|
43
|
+
const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
|
|
44
|
+
this.logger = logger;
|
|
45
|
+
logger.log(`!! using experimentalShardedStream !! ${this.table}, batchSize: ${this.opt.batchSize}`);
|
|
44
46
|
}
|
|
45
47
|
/**
|
|
46
48
|
* Counts how many times _read was called.
|
|
@@ -52,17 +54,17 @@ export class FirestoreShardedReadable extends Readable {
|
|
|
52
54
|
// console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
|
|
53
55
|
this.count++;
|
|
54
56
|
if (this.done) {
|
|
55
|
-
this.logger.
|
|
57
|
+
this.logger.log(`!!! _read was called, but done==true`);
|
|
56
58
|
return;
|
|
57
59
|
}
|
|
58
60
|
// const shard = this.getNextShardAndMove()
|
|
59
61
|
const shard = this.findNextFreeShard();
|
|
60
62
|
if (!shard) {
|
|
61
|
-
this.logger.
|
|
63
|
+
this.logger.debug(`_read ${this.count}: all shards are busy, skipping`);
|
|
62
64
|
return;
|
|
63
65
|
}
|
|
64
66
|
void this.runNextQuery(shard).catch(err => {
|
|
65
|
-
|
|
67
|
+
this.logger.error('error in runNextQuery', err);
|
|
66
68
|
this.emit('error', err);
|
|
67
69
|
});
|
|
68
70
|
}
|
|
@@ -80,7 +82,7 @@ export class FirestoreShardedReadable extends Readable {
|
|
|
80
82
|
if (this.cursorByShard[shard]) {
|
|
81
83
|
q = q.startAfter(this.cursorByShard[shard]);
|
|
82
84
|
}
|
|
83
|
-
|
|
85
|
+
logger.debug(`runNextQuery[${shard}]`, {
|
|
84
86
|
retrieved: this.rowsRetrieved,
|
|
85
87
|
});
|
|
86
88
|
const qs = await this.runQuery(q);
|
|
@@ -98,7 +100,7 @@ export class FirestoreShardedReadable extends Readable {
|
|
|
98
100
|
});
|
|
99
101
|
}
|
|
100
102
|
this.rowsRetrieved += rows.length;
|
|
101
|
-
logger.
|
|
103
|
+
logger.debug(`${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
|
|
102
104
|
this.cursorByShard[shard] = lastDocId;
|
|
103
105
|
this.queryIsRunningByShard[shard] = false; // ready to take more _reads
|
|
104
106
|
this.lastQueryDoneByShard[shard] = localTime.nowUnixMillis();
|
|
@@ -106,18 +108,18 @@ export class FirestoreShardedReadable extends Readable {
|
|
|
106
108
|
this.push(row);
|
|
107
109
|
}
|
|
108
110
|
if (qs.empty) {
|
|
109
|
-
logger.
|
|
111
|
+
logger.log(`!!!! Shard ${shard} DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
|
|
110
112
|
this.doneShards.add(shard);
|
|
111
113
|
}
|
|
112
114
|
if (this.doneShards.size === SHARDS) {
|
|
113
|
-
logger.
|
|
115
|
+
logger.log(`!!!! DONE: all shards completed, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
|
|
114
116
|
this.push(null);
|
|
115
117
|
this.paused = false;
|
|
116
118
|
this.done = true;
|
|
117
119
|
return;
|
|
118
120
|
}
|
|
119
121
|
if (this.originalLimit && this.rowsRetrieved >= this.originalLimit) {
|
|
120
|
-
logger.
|
|
122
|
+
logger.log(`!!!! DONE: reached total limit of ${this.originalLimit}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
|
|
121
123
|
this.push(null);
|
|
122
124
|
this.paused = false;
|
|
123
125
|
this.done = true;
|
|
@@ -131,7 +133,7 @@ export class FirestoreShardedReadable extends Readable {
|
|
|
131
133
|
void this.runNextQuery(nextShard);
|
|
132
134
|
}
|
|
133
135
|
else {
|
|
134
|
-
logger.
|
|
136
|
+
logger.log(`${table} all shards are busy in runNextQuery, skipping`);
|
|
135
137
|
}
|
|
136
138
|
}
|
|
137
139
|
async runQuery(q) {
|
|
@@ -149,7 +151,7 @@ export class FirestoreShardedReadable extends Readable {
|
|
|
149
151
|
});
|
|
150
152
|
}
|
|
151
153
|
catch (err) {
|
|
152
|
-
|
|
154
|
+
logger.error(`FirestoreStreamReadable error!\n`, {
|
|
153
155
|
table,
|
|
154
156
|
rowsRetrieved: this.rowsRetrieved,
|
|
155
157
|
}, err);
|
|
@@ -1,13 +1,11 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
2
|
import { type Query } from '@google-cloud/firestore';
|
|
3
3
|
import type { DBQuery } from '@naturalcycles/db-lib';
|
|
4
|
-
import type { CommonLogger } from '@naturalcycles/js-lib/log';
|
|
5
4
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
6
5
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
7
6
|
import type { FirestoreDBStreamOptions } from './firestore.db.js';
|
|
8
7
|
export declare class FirestoreStreamReadable<T extends ObjectWithId = any> extends Readable implements ReadableTyped<T> {
|
|
9
8
|
private q;
|
|
10
|
-
private logger;
|
|
11
9
|
private readonly table;
|
|
12
10
|
private readonly originalLimit;
|
|
13
11
|
private rowsRetrieved;
|
|
@@ -21,7 +19,8 @@ export declare class FirestoreStreamReadable<T extends ObjectWithId = any> exten
|
|
|
21
19
|
*/
|
|
22
20
|
countReads: number;
|
|
23
21
|
private readonly opt;
|
|
24
|
-
|
|
22
|
+
private logger;
|
|
23
|
+
constructor(q: Query, dbQuery: DBQuery<T>, opt: FirestoreDBStreamOptions);
|
|
25
24
|
_read(): void;
|
|
26
25
|
private runNextQuery;
|
|
27
26
|
private runQuery;
|
|
@@ -2,11 +2,11 @@ import { Readable } from 'node:stream';
|
|
|
2
2
|
import { FieldPath } from '@google-cloud/firestore';
|
|
3
3
|
import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
|
|
4
4
|
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
|
|
5
|
+
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
|
|
5
6
|
import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
|
|
6
7
|
import { unescapeDocId } from './firestore.util.js';
|
|
7
8
|
export class FirestoreStreamReadable extends Readable {
|
|
8
9
|
q;
|
|
9
|
-
logger;
|
|
10
10
|
table;
|
|
11
11
|
originalLimit;
|
|
12
12
|
rowsRetrieved = 0;
|
|
@@ -20,14 +20,14 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
20
20
|
*/
|
|
21
21
|
countReads = 0;
|
|
22
22
|
opt;
|
|
23
|
-
|
|
23
|
+
logger;
|
|
24
|
+
constructor(q, dbQuery, opt) {
|
|
24
25
|
// 10_000 was optimal in benchmarks
|
|
25
26
|
const { batchSize = 10_000 } = opt;
|
|
26
27
|
const { highWaterMark = batchSize * 3 } = opt;
|
|
27
28
|
// Defaulting highWaterMark to 3x batchSize
|
|
28
29
|
super({ objectMode: true, highWaterMark });
|
|
29
30
|
this.q = q;
|
|
30
|
-
this.logger = logger;
|
|
31
31
|
this.opt = {
|
|
32
32
|
...opt,
|
|
33
33
|
batchSize,
|
|
@@ -36,7 +36,9 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
36
36
|
// todo: support PITR!
|
|
37
37
|
this.originalLimit = dbQuery._limitValue;
|
|
38
38
|
this.table = dbQuery.table;
|
|
39
|
-
logger
|
|
39
|
+
const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
|
|
40
|
+
this.logger = logger;
|
|
41
|
+
logger.log(`!!! using experimentalCursorStream`, {
|
|
40
42
|
table: this.table,
|
|
41
43
|
batchSize,
|
|
42
44
|
highWaterMark,
|
|
@@ -55,11 +57,11 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
55
57
|
this.paused = false;
|
|
56
58
|
}
|
|
57
59
|
if (this.queryIsRunning) {
|
|
58
|
-
this.logger.
|
|
60
|
+
this.logger.debug(`_read #${this.countReads}, queryIsRunning: true, doing nothing`);
|
|
59
61
|
return;
|
|
60
62
|
}
|
|
61
63
|
void this.runNextQuery().catch(err => {
|
|
62
|
-
|
|
64
|
+
this.logger.error('error in runNextQuery', err);
|
|
63
65
|
this.emit('error', err);
|
|
64
66
|
});
|
|
65
67
|
}
|
|
@@ -98,7 +100,7 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
98
100
|
});
|
|
99
101
|
}
|
|
100
102
|
this.rowsRetrieved += rows.length;
|
|
101
|
-
logger.
|
|
103
|
+
logger.debug(`${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved`);
|
|
102
104
|
this.endCursor = lastDocId;
|
|
103
105
|
this.queryIsRunning = false; // ready to take more _reads
|
|
104
106
|
let shouldContinue = false;
|
|
@@ -106,7 +108,7 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
106
108
|
shouldContinue = this.push(row);
|
|
107
109
|
}
|
|
108
110
|
if (!rows.length || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
|
|
109
|
-
logger.
|
|
111
|
+
logger.log(`${table} DONE! ${this.rowsRetrieved} rowsRetrieved`);
|
|
110
112
|
this.push(null);
|
|
111
113
|
this.done = true;
|
|
112
114
|
this.paused = false;
|
|
@@ -114,13 +116,13 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
114
116
|
}
|
|
115
117
|
if (shouldContinue) {
|
|
116
118
|
// Keep the stream flowing
|
|
117
|
-
logger.
|
|
119
|
+
logger.debug(`${table} continuing the stream`);
|
|
118
120
|
void this.runNextQuery();
|
|
119
121
|
}
|
|
120
122
|
else {
|
|
121
123
|
// Not starting the next query
|
|
122
124
|
if (this.paused) {
|
|
123
|
-
logger.
|
|
125
|
+
logger.debug(`${table} stream is already paused`);
|
|
124
126
|
}
|
|
125
127
|
else {
|
|
126
128
|
logger.log(`${table} pausing the stream`);
|
|
@@ -143,7 +145,7 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
143
145
|
});
|
|
144
146
|
}
|
|
145
147
|
catch (err) {
|
|
146
|
-
|
|
148
|
+
logger.error(`FirestoreStreamReadable error!\n`, {
|
|
147
149
|
table,
|
|
148
150
|
rowsRetrieved: this.rowsRetrieved,
|
|
149
151
|
}, err);
|
package/package.json
CHANGED
package/src/firestore.db.ts
CHANGED
|
@@ -25,7 +25,7 @@ import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib'
|
|
|
25
25
|
import { _isTruthy } from '@naturalcycles/js-lib'
|
|
26
26
|
import { _chunk } from '@naturalcycles/js-lib/array/array.util.js'
|
|
27
27
|
import { _assert } from '@naturalcycles/js-lib/error/assert.js'
|
|
28
|
-
import {
|
|
28
|
+
import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log'
|
|
29
29
|
import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/object.util.js'
|
|
30
30
|
import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
|
|
31
31
|
import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types'
|
|
@@ -156,26 +156,18 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
|
156
156
|
const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table))
|
|
157
157
|
|
|
158
158
|
const opt: FirestoreDBStreamOptions = {
|
|
159
|
+
logger: this.cfg.logger,
|
|
160
|
+
logLevel: this.cfg.logLevel,
|
|
159
161
|
...this.cfg.streamOptions,
|
|
160
162
|
...opt_,
|
|
161
163
|
}
|
|
162
164
|
|
|
163
165
|
if (opt.experimentalCursorStream) {
|
|
164
|
-
return new FirestoreStreamReadable(
|
|
165
|
-
firestoreQuery,
|
|
166
|
-
q,
|
|
167
|
-
opt,
|
|
168
|
-
commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
|
|
169
|
-
)
|
|
166
|
+
return new FirestoreStreamReadable(firestoreQuery, q, opt)
|
|
170
167
|
}
|
|
171
168
|
|
|
172
169
|
if (opt.experimentalShardedStream) {
|
|
173
|
-
return new FirestoreShardedReadable(
|
|
174
|
-
firestoreQuery,
|
|
175
|
-
q,
|
|
176
|
-
opt,
|
|
177
|
-
commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
|
|
178
|
-
)
|
|
170
|
+
return new FirestoreShardedReadable(firestoreQuery, q, opt)
|
|
179
171
|
}
|
|
180
172
|
|
|
181
173
|
return (firestoreQuery.stream() as ReadableTyped<QueryDocumentSnapshot<any>>).map(doc => {
|
|
@@ -534,6 +526,8 @@ export interface FirestoreDBCfg {
|
|
|
534
526
|
* Default to `console`
|
|
535
527
|
*/
|
|
536
528
|
logger?: CommonLogger
|
|
529
|
+
|
|
530
|
+
logLevel?: CommonLogLevel
|
|
537
531
|
}
|
|
538
532
|
|
|
539
533
|
const methodMap: Record<CommonDBSaveMethod, SaveOp> = {
|
|
@@ -575,12 +569,13 @@ export interface FirestoreDBStreamOptions extends FirestoreDBReadOptions {
|
|
|
575
569
|
*/
|
|
576
570
|
highWaterMark?: PositiveInteger
|
|
577
571
|
|
|
572
|
+
logger?: CommonLogger
|
|
573
|
+
|
|
578
574
|
/**
|
|
579
|
-
*
|
|
580
|
-
*
|
|
581
|
-
* @default false
|
|
575
|
+
* Defaults to `log`.
|
|
576
|
+
* Set to `debug` to allow for extra debugging, e.g in experimentalCursorStream.
|
|
582
577
|
*/
|
|
583
|
-
|
|
578
|
+
logLevel?: CommonLogLevel
|
|
584
579
|
}
|
|
585
580
|
|
|
586
581
|
export interface FirestoreDBOptions extends CommonDBOptions {}
|
|
@@ -3,7 +3,7 @@ import { FieldPath, type Query, type QuerySnapshot } from '@google-cloud/firesto
|
|
|
3
3
|
import type { DBQuery } from '@naturalcycles/db-lib'
|
|
4
4
|
import { localTime } from '@naturalcycles/js-lib/datetime'
|
|
5
5
|
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
|
|
6
|
-
import type
|
|
6
|
+
import { type CommonLogger, createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
|
|
7
7
|
import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
|
|
8
8
|
import type {
|
|
9
9
|
ObjectWithId,
|
|
@@ -42,12 +42,12 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
42
42
|
private totalWait = 0
|
|
43
43
|
|
|
44
44
|
private readonly opt: FirestoreDBStreamOptions & { batchSize: number }
|
|
45
|
+
private logger: CommonLogger
|
|
45
46
|
|
|
46
47
|
constructor(
|
|
47
48
|
private readonly q: Query,
|
|
48
49
|
readonly dbQuery: DBQuery<T>,
|
|
49
50
|
opt: FirestoreDBStreamOptions,
|
|
50
|
-
private logger: CommonLogger,
|
|
51
51
|
) {
|
|
52
52
|
super({ objectMode: true })
|
|
53
53
|
|
|
@@ -58,8 +58,10 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
58
58
|
|
|
59
59
|
this.originalLimit = dbQuery._limitValue
|
|
60
60
|
this.table = dbQuery.table
|
|
61
|
+
const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
|
|
62
|
+
this.logger = logger
|
|
61
63
|
|
|
62
|
-
logger.
|
|
64
|
+
logger.log(
|
|
63
65
|
`!! using experimentalShardedStream !! ${this.table}, batchSize: ${this.opt.batchSize}`,
|
|
64
66
|
)
|
|
65
67
|
}
|
|
@@ -77,18 +79,18 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
77
79
|
this.count++
|
|
78
80
|
|
|
79
81
|
if (this.done) {
|
|
80
|
-
this.logger.
|
|
82
|
+
this.logger.log(`!!! _read was called, but done==true`)
|
|
81
83
|
return
|
|
82
84
|
}
|
|
83
85
|
|
|
84
86
|
// const shard = this.getNextShardAndMove()
|
|
85
87
|
const shard = this.findNextFreeShard()
|
|
86
88
|
if (!shard) {
|
|
87
|
-
this.logger.
|
|
89
|
+
this.logger.debug(`_read ${this.count}: all shards are busy, skipping`)
|
|
88
90
|
return
|
|
89
91
|
}
|
|
90
92
|
void this.runNextQuery(shard).catch(err => {
|
|
91
|
-
|
|
93
|
+
this.logger.error('error in runNextQuery', err)
|
|
92
94
|
this.emit('error', err)
|
|
93
95
|
})
|
|
94
96
|
}
|
|
@@ -112,7 +114,7 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
112
114
|
q = q.startAfter(this.cursorByShard[shard])
|
|
113
115
|
}
|
|
114
116
|
|
|
115
|
-
|
|
117
|
+
logger.debug(`runNextQuery[${shard}]`, {
|
|
116
118
|
retrieved: this.rowsRetrieved,
|
|
117
119
|
})
|
|
118
120
|
const qs = await this.runQuery(q)
|
|
@@ -133,7 +135,7 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
133
135
|
}
|
|
134
136
|
|
|
135
137
|
this.rowsRetrieved += rows.length
|
|
136
|
-
logger.
|
|
138
|
+
logger.debug(
|
|
137
139
|
`${table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
|
|
138
140
|
this.totalWait,
|
|
139
141
|
)}`,
|
|
@@ -148,14 +150,14 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
148
150
|
}
|
|
149
151
|
|
|
150
152
|
if (qs.empty) {
|
|
151
|
-
logger.
|
|
153
|
+
logger.log(
|
|
152
154
|
`!!!! Shard ${shard} DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
|
|
153
155
|
)
|
|
154
156
|
this.doneShards.add(shard)
|
|
155
157
|
}
|
|
156
158
|
|
|
157
159
|
if (this.doneShards.size === SHARDS) {
|
|
158
|
-
logger.
|
|
160
|
+
logger.log(
|
|
159
161
|
`!!!! DONE: all shards completed, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
|
|
160
162
|
)
|
|
161
163
|
this.push(null)
|
|
@@ -165,7 +167,7 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
165
167
|
}
|
|
166
168
|
|
|
167
169
|
if (this.originalLimit && this.rowsRetrieved >= this.originalLimit) {
|
|
168
|
-
logger.
|
|
170
|
+
logger.log(
|
|
169
171
|
`!!!! DONE: reached total limit of ${this.originalLimit}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
|
|
170
172
|
)
|
|
171
173
|
this.push(null)
|
|
@@ -181,7 +183,7 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
181
183
|
if (nextShard) {
|
|
182
184
|
void this.runNextQuery(nextShard)
|
|
183
185
|
} else {
|
|
184
|
-
logger.
|
|
186
|
+
logger.log(`${table} all shards are busy in runNextQuery, skipping`)
|
|
185
187
|
}
|
|
186
188
|
}
|
|
187
189
|
|
|
@@ -203,7 +205,7 @@ export class FirestoreShardedReadable<T extends ObjectWithId = any>
|
|
|
203
205
|
},
|
|
204
206
|
)
|
|
205
207
|
} catch (err) {
|
|
206
|
-
|
|
208
|
+
logger.error(
|
|
207
209
|
`FirestoreStreamReadable error!\n`,
|
|
208
210
|
{
|
|
209
211
|
table,
|
|
@@ -3,7 +3,7 @@ import { FieldPath, type Query, type QuerySnapshot } from '@google-cloud/firesto
|
|
|
3
3
|
import type { DBQuery } from '@naturalcycles/db-lib'
|
|
4
4
|
import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
|
|
5
5
|
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
|
|
6
|
-
import type
|
|
6
|
+
import { type CommonLogger, createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
|
|
7
7
|
import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
|
|
8
8
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types'
|
|
9
9
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
@@ -28,12 +28,12 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
28
28
|
countReads = 0
|
|
29
29
|
|
|
30
30
|
private readonly opt: FirestoreDBStreamOptions & { batchSize: number; highWaterMark: number }
|
|
31
|
+
private logger: CommonLogger
|
|
31
32
|
|
|
32
33
|
constructor(
|
|
33
34
|
private q: Query,
|
|
34
35
|
dbQuery: DBQuery<T>,
|
|
35
36
|
opt: FirestoreDBStreamOptions,
|
|
36
|
-
private logger: CommonLogger,
|
|
37
37
|
) {
|
|
38
38
|
// 10_000 was optimal in benchmarks
|
|
39
39
|
const { batchSize = 10_000 } = opt
|
|
@@ -50,8 +50,10 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
50
50
|
|
|
51
51
|
this.originalLimit = dbQuery._limitValue
|
|
52
52
|
this.table = dbQuery.table
|
|
53
|
+
const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
|
|
54
|
+
this.logger = logger
|
|
53
55
|
|
|
54
|
-
logger.
|
|
56
|
+
logger.log(`!!! using experimentalCursorStream`, {
|
|
55
57
|
table: this.table,
|
|
56
58
|
batchSize,
|
|
57
59
|
highWaterMark,
|
|
@@ -77,12 +79,12 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
77
79
|
}
|
|
78
80
|
|
|
79
81
|
if (this.queryIsRunning) {
|
|
80
|
-
this.logger.
|
|
82
|
+
this.logger.debug(`_read #${this.countReads}, queryIsRunning: true, doing nothing`)
|
|
81
83
|
return
|
|
82
84
|
}
|
|
83
85
|
|
|
84
86
|
void this.runNextQuery().catch(err => {
|
|
85
|
-
|
|
87
|
+
this.logger.error('error in runNextQuery', err)
|
|
86
88
|
this.emit('error', err)
|
|
87
89
|
})
|
|
88
90
|
}
|
|
@@ -130,7 +132,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
130
132
|
}
|
|
131
133
|
|
|
132
134
|
this.rowsRetrieved += rows.length
|
|
133
|
-
logger.
|
|
135
|
+
logger.debug(
|
|
134
136
|
`${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved`,
|
|
135
137
|
)
|
|
136
138
|
|
|
@@ -143,7 +145,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
143
145
|
}
|
|
144
146
|
|
|
145
147
|
if (!rows.length || (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
|
|
146
|
-
logger.
|
|
148
|
+
logger.log(`${table} DONE! ${this.rowsRetrieved} rowsRetrieved`)
|
|
147
149
|
this.push(null)
|
|
148
150
|
this.done = true
|
|
149
151
|
this.paused = false
|
|
@@ -152,12 +154,12 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
152
154
|
|
|
153
155
|
if (shouldContinue) {
|
|
154
156
|
// Keep the stream flowing
|
|
155
|
-
logger.
|
|
157
|
+
logger.debug(`${table} continuing the stream`)
|
|
156
158
|
void this.runNextQuery()
|
|
157
159
|
} else {
|
|
158
160
|
// Not starting the next query
|
|
159
161
|
if (this.paused) {
|
|
160
|
-
logger.
|
|
162
|
+
logger.debug(`${table} stream is already paused`)
|
|
161
163
|
} else {
|
|
162
164
|
logger.log(`${table} pausing the stream`)
|
|
163
165
|
this.paused = true
|
|
@@ -183,7 +185,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
183
185
|
},
|
|
184
186
|
)
|
|
185
187
|
} catch (err) {
|
|
186
|
-
|
|
188
|
+
logger.error(
|
|
187
189
|
`FirestoreStreamReadable error!\n`,
|
|
188
190
|
{
|
|
189
191
|
table,
|