@naturalcycles/firestore-lib 2.14.3 → 2.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/firestore.db.d.ts +3 -3
- package/dist/firestore.db.js +43 -14
- package/dist/firestoreShardedReadable.d.ts +1 -1
- package/dist/firestoreStreamReadable.d.ts +2 -1
- package/dist/firestoreStreamReadable.js +13 -3
- package/dist/query.util.d.ts +4 -2
- package/dist/query.util.js +13 -5
- package/package.json +2 -2
- package/src/firestore.db.ts +49 -15
- package/src/firestoreShardedReadable.ts +4 -2
- package/src/firestoreStreamReadable.ts +21 -8
- package/src/query.util.ts +16 -2
package/dist/firestore.db.d.ts
CHANGED
|
@@ -12,9 +12,9 @@ export declare class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
|
12
12
|
support: CommonDBSupport;
|
|
13
13
|
getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: FirestoreDBReadOptions): Promise<ROW[]>;
|
|
14
14
|
multiGet<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: CommonDBReadOptions): Promise<StringMap<ROW[]>>;
|
|
15
|
-
runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?:
|
|
16
|
-
runFirestoreQuery<ROW extends ObjectWithId>(q: Query): Promise<ROW[]>;
|
|
17
|
-
runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>,
|
|
15
|
+
runQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: FirestoreDBReadOptions): Promise<RunQueryResult<ROW>>;
|
|
16
|
+
runFirestoreQuery<ROW extends ObjectWithId>(q: Query, opt?: FirestoreDBReadOptions): Promise<ROW[]>;
|
|
17
|
+
runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: FirestoreDBReadOptions): Promise<number>;
|
|
18
18
|
streamQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt_?: FirestoreDBStreamOptions): Pipeline<ROW>;
|
|
19
19
|
saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
|
|
20
20
|
multiSave<ROW extends ObjectWithId>(map: StringMap<ROW[]>, opt?: FirestoreDBSaveOptions<ROW>): Promise<void>;
|
package/dist/firestore.db.js
CHANGED
|
@@ -10,7 +10,7 @@ import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
|
|
|
10
10
|
import { escapeDocId, unescapeDocId } from './firestore.util.js';
|
|
11
11
|
import { FirestoreShardedReadable } from './firestoreShardedReadable.js';
|
|
12
12
|
import { FirestoreStreamReadable } from './firestoreStreamReadable.js';
|
|
13
|
-
import { dbQueryToFirestoreQuery } from './query.util.js';
|
|
13
|
+
import { dbQueryToFirestoreQuery, readAtToReadTime } from './query.util.js';
|
|
14
14
|
export class FirestoreDB extends BaseCommonDB {
|
|
15
15
|
constructor(cfg) {
|
|
16
16
|
super();
|
|
@@ -30,10 +30,17 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
30
30
|
async getByIds(table, ids, opt = {}) {
|
|
31
31
|
if (!ids.length)
|
|
32
32
|
return [];
|
|
33
|
-
// todo: support PITR: https://firebase.google.com/docs/firestore/enterprise/use-pitr#read-pitr
|
|
34
33
|
const { firestore } = this.cfg;
|
|
35
34
|
const col = firestore.collection(table);
|
|
36
|
-
|
|
35
|
+
const refs = ids.map(id => col.doc(escapeDocId(id)));
|
|
36
|
+
const readTime = readAtToReadTime(opt);
|
|
37
|
+
if (readTime) {
|
|
38
|
+
_assert(!opt.tx, 'readAt is not supported inside an existing transaction');
|
|
39
|
+
}
|
|
40
|
+
const snapshots = await (readTime
|
|
41
|
+
? firestore.runTransaction(tx => tx.getAll(...refs), { readOnly: true, readTime })
|
|
42
|
+
: (opt.tx?.tx || firestore).getAll(...refs));
|
|
43
|
+
return snapshots
|
|
37
44
|
.map(doc => {
|
|
38
45
|
const data = doc.data();
|
|
39
46
|
if (data === undefined)
|
|
@@ -54,7 +61,13 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
54
61
|
const col = firestore.collection(table);
|
|
55
62
|
refs.push(...ids.map(id => col.doc(escapeDocId(id))));
|
|
56
63
|
}
|
|
57
|
-
const
|
|
64
|
+
const readTime = readAtToReadTime(opt);
|
|
65
|
+
if (readTime) {
|
|
66
|
+
_assert(!opt.tx, 'readAt is not supported inside an existing transaction');
|
|
67
|
+
}
|
|
68
|
+
const snapshots = await (readTime
|
|
69
|
+
? firestore.runTransaction(tx => tx.getAll(...refs), { readOnly: true, readTime })
|
|
70
|
+
: (opt.tx?.tx || firestore).getAll(...refs));
|
|
58
71
|
snapshots.forEach(snap => {
|
|
59
72
|
const data = snap.data();
|
|
60
73
|
if (data === undefined)
|
|
@@ -69,7 +82,7 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
69
82
|
return result;
|
|
70
83
|
}
|
|
71
84
|
// QUERY
|
|
72
|
-
async runQuery(q, opt) {
|
|
85
|
+
async runQuery(q, opt = {}) {
|
|
73
86
|
const idFilter = q._filters.find(f => f.name === 'id');
|
|
74
87
|
if (idFilter) {
|
|
75
88
|
const ids = Array.isArray(idFilter.val) ? idFilter.val : [idFilter.val];
|
|
@@ -78,18 +91,34 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
78
91
|
};
|
|
79
92
|
}
|
|
80
93
|
const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table));
|
|
81
|
-
let rows = await this.runFirestoreQuery(firestoreQuery);
|
|
94
|
+
let rows = await this.runFirestoreQuery(firestoreQuery, opt);
|
|
82
95
|
// Special case when projection query didn't specify 'id'
|
|
83
96
|
if (q._selectedFieldNames && !q._selectedFieldNames.includes('id')) {
|
|
84
97
|
rows = rows.map(r => _omit(r, ['id']));
|
|
85
98
|
}
|
|
86
99
|
return { rows };
|
|
87
100
|
}
|
|
88
|
-
async runFirestoreQuery(q) {
|
|
101
|
+
async runFirestoreQuery(q, opt = {}) {
|
|
102
|
+
const readTime = readAtToReadTime(opt);
|
|
103
|
+
if (readTime) {
|
|
104
|
+
const qs = await this.cfg.firestore.runTransaction(tx => tx.get(q), {
|
|
105
|
+
readOnly: true,
|
|
106
|
+
readTime,
|
|
107
|
+
});
|
|
108
|
+
return this.querySnapshotToArray(qs);
|
|
109
|
+
}
|
|
89
110
|
return this.querySnapshotToArray(await q.get());
|
|
90
111
|
}
|
|
91
|
-
async runQueryCount(q,
|
|
112
|
+
async runQueryCount(q, opt = {}) {
|
|
92
113
|
const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table));
|
|
114
|
+
const readTime = readAtToReadTime(opt);
|
|
115
|
+
if (readTime) {
|
|
116
|
+
const r = await this.cfg.firestore.runTransaction(tx => tx.get(firestoreQuery.count()), {
|
|
117
|
+
readOnly: true,
|
|
118
|
+
readTime,
|
|
119
|
+
});
|
|
120
|
+
return r.data().count;
|
|
121
|
+
}
|
|
93
122
|
const r = await firestoreQuery.count().get();
|
|
94
123
|
return r.data().count;
|
|
95
124
|
}
|
|
@@ -101,7 +130,7 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
101
130
|
...this.cfg.streamOptions,
|
|
102
131
|
...opt_,
|
|
103
132
|
};
|
|
104
|
-
if (opt.experimentalCursorStream) {
|
|
133
|
+
if (opt.experimentalCursorStream || opt.readAt) {
|
|
105
134
|
return Pipeline.from(new FirestoreStreamReadable(firestoreQuery, q, opt));
|
|
106
135
|
}
|
|
107
136
|
if (opt.experimentalShardedStream) {
|
|
@@ -128,7 +157,7 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
128
157
|
}
|
|
129
158
|
return;
|
|
130
159
|
}
|
|
131
|
-
await pMap(_chunk(rows, MAX_ITEMS), async
|
|
160
|
+
await pMap(_chunk(rows, MAX_ITEMS), async chunk => {
|
|
132
161
|
// .batch is called "Atomic batch writer"
|
|
133
162
|
// Executes multiple writes in a single atomic transaction-like commit — all succeed or all fail.
|
|
134
163
|
// If any write in the batch fails (e.g., permission error, missing doc), the whole batch fails.
|
|
@@ -163,7 +192,7 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
163
192
|
tableRows.push([table, row]);
|
|
164
193
|
}
|
|
165
194
|
}
|
|
166
|
-
await pMap(_chunk(tableRows, MAX_ITEMS), async
|
|
195
|
+
await pMap(_chunk(tableRows, MAX_ITEMS), async chunk => {
|
|
167
196
|
const batch = firestore.batch();
|
|
168
197
|
for (const [table, row] of chunk) {
|
|
169
198
|
_assert(row.id, `firestore-db doesn't support id auto-generation, but empty id was provided in multiSaveBatch`);
|
|
@@ -207,7 +236,7 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
207
236
|
}
|
|
208
237
|
return ids.length;
|
|
209
238
|
}
|
|
210
|
-
await pMap(_chunk(ids, MAX_ITEMS), async
|
|
239
|
+
await pMap(_chunk(ids, MAX_ITEMS), async chunk => {
|
|
211
240
|
const batch = firestore.batch();
|
|
212
241
|
for (const id of chunk) {
|
|
213
242
|
batch.delete(col.doc(escapeDocId(id)));
|
|
@@ -230,7 +259,7 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
230
259
|
}
|
|
231
260
|
}
|
|
232
261
|
else {
|
|
233
|
-
await pMap(_chunk(refs, MAX_ITEMS), async
|
|
262
|
+
await pMap(_chunk(refs, MAX_ITEMS), async chunk => {
|
|
234
263
|
const batch = firestore.batch();
|
|
235
264
|
for (const ref of chunk) {
|
|
236
265
|
batch.delete(ref);
|
|
@@ -249,7 +278,7 @@ export class FirestoreDB extends BaseCommonDB {
|
|
|
249
278
|
async runInTransaction(fn, opt = {}) {
|
|
250
279
|
const { readOnly } = opt;
|
|
251
280
|
try {
|
|
252
|
-
await this.cfg.firestore.runTransaction(async
|
|
281
|
+
await this.cfg.firestore.runTransaction(async firestoreTx => {
|
|
253
282
|
const tx = new FirestoreDBTransaction(this, firestoreTx);
|
|
254
283
|
await fn(tx);
|
|
255
284
|
}, {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
|
-
import {
|
|
2
|
+
import type { Query } from '@google-cloud/firestore';
|
|
3
3
|
import type { DBQuery } from '@naturalcycles/db-lib';
|
|
4
4
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
5
5
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
|
-
import {
|
|
2
|
+
import type { Query } from '@google-cloud/firestore';
|
|
3
3
|
import type { DBQuery } from '@naturalcycles/db-lib';
|
|
4
4
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
5
5
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
|
|
@@ -18,6 +18,7 @@ export declare class FirestoreStreamReadable<T extends ObjectWithId = any> exten
|
|
|
18
18
|
* For debugging.
|
|
19
19
|
*/
|
|
20
20
|
countReads: number;
|
|
21
|
+
private readonly readTime?;
|
|
21
22
|
private readonly opt;
|
|
22
23
|
private logger;
|
|
23
24
|
constructor(q: Query, dbQuery: DBQuery<T>, opt: FirestoreDBStreamOptions);
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { Readable } from 'node:stream';
|
|
2
|
-
import { FieldPath
|
|
2
|
+
import { FieldPath } from '@google-cloud/firestore';
|
|
3
3
|
import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
|
|
4
4
|
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
|
|
5
|
+
import { TimeoutError } from '@naturalcycles/js-lib/error/error.util.js';
|
|
5
6
|
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
|
|
6
7
|
import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
|
|
7
8
|
import { unescapeDocId } from './firestore.util.js';
|
|
9
|
+
import { readAtToReadTime } from './query.util.js';
|
|
8
10
|
export class FirestoreStreamReadable extends Readable {
|
|
9
11
|
q;
|
|
10
12
|
table;
|
|
@@ -19,6 +21,7 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
19
21
|
* For debugging.
|
|
20
22
|
*/
|
|
21
23
|
countReads = 0;
|
|
24
|
+
readTime;
|
|
22
25
|
opt;
|
|
23
26
|
logger;
|
|
24
27
|
constructor(q, dbQuery, opt) {
|
|
@@ -33,7 +36,7 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
33
36
|
batchSize,
|
|
34
37
|
highWaterMark,
|
|
35
38
|
};
|
|
36
|
-
|
|
39
|
+
this.readTime = readAtToReadTime(opt);
|
|
37
40
|
this.originalLimit = dbQuery._limitValue;
|
|
38
41
|
this.table = dbQuery.table;
|
|
39
42
|
const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
|
|
@@ -134,10 +137,17 @@ export class FirestoreStreamReadable extends Readable {
|
|
|
134
137
|
const { table, logger } = this;
|
|
135
138
|
try {
|
|
136
139
|
return await pRetry(async () => {
|
|
140
|
+
if (this.readTime) {
|
|
141
|
+
return await this.q.firestore.runTransaction(tx => tx.get(q), {
|
|
142
|
+
readOnly: true,
|
|
143
|
+
readTime: this.readTime,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
137
146
|
return await q.get();
|
|
138
147
|
}, {
|
|
139
148
|
name: `FirestoreStreamReadable.query(${table})`,
|
|
140
|
-
predicate: err =>
|
|
149
|
+
predicate: err => err instanceof TimeoutError ||
|
|
150
|
+
RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
|
|
141
151
|
maxAttempts: 5,
|
|
142
152
|
delay: 5000,
|
|
143
153
|
delayMultiplier: 2,
|
package/dist/query.util.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import type {
|
|
1
|
+
import { Timestamp } from '@google-cloud/firestore';
|
|
2
|
+
import type { Query } from '@google-cloud/firestore';
|
|
3
|
+
import type { CommonDBReadOptions, DBQuery } from '@naturalcycles/db-lib';
|
|
3
4
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types';
|
|
5
|
+
export declare function readAtToReadTime(opt: CommonDBReadOptions): Timestamp | undefined;
|
|
4
6
|
export declare function dbQueryToFirestoreQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, emptyQuery: Query): Query;
|
package/dist/query.util.js
CHANGED
|
@@ -1,10 +1,18 @@
|
|
|
1
|
-
import { FieldPath } from '@google-cloud/firestore';
|
|
1
|
+
import { FieldPath, Timestamp } from '@google-cloud/firestore';
|
|
2
|
+
import { _round } from '@naturalcycles/js-lib';
|
|
3
|
+
export function readAtToReadTime(opt) {
|
|
4
|
+
if (!opt.readAt)
|
|
5
|
+
return;
|
|
6
|
+
// Same logic as Datastore: round to whole minutes, guard against future
|
|
7
|
+
let readTimeMs = _round(opt.readAt, 60) * 1000;
|
|
8
|
+
if (readTimeMs >= Date.now() - 1000) {
|
|
9
|
+
readTimeMs -= 60_000;
|
|
10
|
+
}
|
|
11
|
+
return Timestamp.fromMillis(readTimeMs);
|
|
12
|
+
}
|
|
2
13
|
// Map DBQueryFilterOp to WhereFilterOp
|
|
3
14
|
// Currently it's fully aligned!
|
|
4
|
-
const OP_MAP = {
|
|
5
|
-
// '=': '==',
|
|
6
|
-
// in: 'array-contains',
|
|
7
|
-
};
|
|
15
|
+
const OP_MAP = {};
|
|
8
16
|
export function dbQueryToFirestoreQuery(dbQuery, emptyQuery) {
|
|
9
17
|
let q = emptyQuery;
|
|
10
18
|
// filter
|
package/package.json
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
},
|
|
11
11
|
"devDependencies": {
|
|
12
12
|
"@types/node": "^25",
|
|
13
|
-
"
|
|
13
|
+
"@typescript/native-preview": "7.0.0-dev.20260301.1",
|
|
14
14
|
"firebase-admin": "^13",
|
|
15
15
|
"@naturalcycles/dev-lib": "18.4.2"
|
|
16
16
|
},
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
"engines": {
|
|
39
39
|
"node": ">=24.10.0"
|
|
40
40
|
},
|
|
41
|
-
"version": "2.
|
|
41
|
+
"version": "2.15.0",
|
|
42
42
|
"description": "Firestore implementation of CommonDB interface",
|
|
43
43
|
"author": "Natural Cycles Team",
|
|
44
44
|
"license": "MIT",
|
package/src/firestore.db.ts
CHANGED
|
@@ -30,11 +30,12 @@ import { _filterUndefinedValues, _omit } from '@naturalcycles/js-lib/object/obje
|
|
|
30
30
|
import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
|
|
31
31
|
import type { ObjectWithId, PositiveInteger, StringMap } from '@naturalcycles/js-lib/types'
|
|
32
32
|
import { _stringMapEntries } from '@naturalcycles/js-lib/types'
|
|
33
|
-
import { Pipeline
|
|
33
|
+
import { Pipeline } from '@naturalcycles/nodejs-lib/stream'
|
|
34
|
+
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
34
35
|
import { escapeDocId, unescapeDocId } from './firestore.util.js'
|
|
35
36
|
import { FirestoreShardedReadable } from './firestoreShardedReadable.js'
|
|
36
37
|
import { FirestoreStreamReadable } from './firestoreStreamReadable.js'
|
|
37
|
-
import { dbQueryToFirestoreQuery } from './query.util.js'
|
|
38
|
+
import { dbQueryToFirestoreQuery, readAtToReadTime } from './query.util.js'
|
|
38
39
|
|
|
39
40
|
export class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
40
41
|
constructor(cfg: FirestoreDBCfg) {
|
|
@@ -62,16 +63,20 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
|
62
63
|
): Promise<ROW[]> {
|
|
63
64
|
if (!ids.length) return []
|
|
64
65
|
|
|
65
|
-
// todo: support PITR: https://firebase.google.com/docs/firestore/enterprise/use-pitr#read-pitr
|
|
66
|
-
|
|
67
66
|
const { firestore } = this.cfg
|
|
68
67
|
const col = firestore.collection(table)
|
|
68
|
+
const refs = ids.map(id => col.doc(escapeDocId(id)))
|
|
69
69
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
70
|
+
const readTime = readAtToReadTime(opt)
|
|
71
|
+
if (readTime) {
|
|
72
|
+
_assert(!opt.tx, 'readAt is not supported inside an existing transaction')
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const snapshots = await (readTime
|
|
76
|
+
? firestore.runTransaction(tx => tx.getAll(...refs), { readOnly: true, readTime })
|
|
77
|
+
: ((opt.tx as FirestoreDBTransaction)?.tx || firestore).getAll(...refs))
|
|
78
|
+
|
|
79
|
+
return snapshots
|
|
75
80
|
.map(doc => {
|
|
76
81
|
const data = doc.data()
|
|
77
82
|
if (data === undefined) return
|
|
@@ -96,7 +101,15 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
|
96
101
|
refs.push(...ids.map(id => col.doc(escapeDocId(id))))
|
|
97
102
|
}
|
|
98
103
|
|
|
99
|
-
const
|
|
104
|
+
const readTime = readAtToReadTime(opt)
|
|
105
|
+
if (readTime) {
|
|
106
|
+
_assert(!opt.tx, 'readAt is not supported inside an existing transaction')
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const snapshots = await (readTime
|
|
110
|
+
? firestore.runTransaction(tx => tx.getAll(...refs), { readOnly: true, readTime })
|
|
111
|
+
: ((opt.tx as FirestoreDBTransaction)?.tx || firestore).getAll(...refs))
|
|
112
|
+
|
|
100
113
|
snapshots.forEach(snap => {
|
|
101
114
|
const data = snap.data()
|
|
102
115
|
if (data === undefined) return
|
|
@@ -114,7 +127,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
|
114
127
|
// QUERY
|
|
115
128
|
override async runQuery<ROW extends ObjectWithId>(
|
|
116
129
|
q: DBQuery<ROW>,
|
|
117
|
-
opt
|
|
130
|
+
opt: FirestoreDBReadOptions = {},
|
|
118
131
|
): Promise<RunQueryResult<ROW>> {
|
|
119
132
|
const idFilter = q._filters.find(f => f.name === 'id')
|
|
120
133
|
if (idFilter) {
|
|
@@ -126,7 +139,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
|
126
139
|
|
|
127
140
|
const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table))
|
|
128
141
|
|
|
129
|
-
let rows = await this.runFirestoreQuery<ROW>(firestoreQuery)
|
|
142
|
+
let rows = await this.runFirestoreQuery<ROW>(firestoreQuery, opt)
|
|
130
143
|
|
|
131
144
|
// Special case when projection query didn't specify 'id'
|
|
132
145
|
if (q._selectedFieldNames && !q._selectedFieldNames.includes('id')) {
|
|
@@ -136,15 +149,36 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
|
136
149
|
return { rows }
|
|
137
150
|
}
|
|
138
151
|
|
|
139
|
-
async runFirestoreQuery<ROW extends ObjectWithId>(
|
|
152
|
+
async runFirestoreQuery<ROW extends ObjectWithId>(
|
|
153
|
+
q: Query,
|
|
154
|
+
opt: FirestoreDBReadOptions = {},
|
|
155
|
+
): Promise<ROW[]> {
|
|
156
|
+
const readTime = readAtToReadTime(opt)
|
|
157
|
+
if (readTime) {
|
|
158
|
+
const qs = await this.cfg.firestore.runTransaction(tx => tx.get(q), {
|
|
159
|
+
readOnly: true,
|
|
160
|
+
readTime,
|
|
161
|
+
})
|
|
162
|
+
return this.querySnapshotToArray(qs)
|
|
163
|
+
}
|
|
140
164
|
return this.querySnapshotToArray(await q.get())
|
|
141
165
|
}
|
|
142
166
|
|
|
143
167
|
override async runQueryCount<ROW extends ObjectWithId>(
|
|
144
168
|
q: DBQuery<ROW>,
|
|
145
|
-
|
|
169
|
+
opt: FirestoreDBReadOptions = {},
|
|
146
170
|
): Promise<number> {
|
|
147
171
|
const firestoreQuery = dbQueryToFirestoreQuery(q, this.cfg.firestore.collection(q.table))
|
|
172
|
+
|
|
173
|
+
const readTime = readAtToReadTime(opt)
|
|
174
|
+
if (readTime) {
|
|
175
|
+
const r = await this.cfg.firestore.runTransaction(tx => tx.get(firestoreQuery.count()), {
|
|
176
|
+
readOnly: true,
|
|
177
|
+
readTime,
|
|
178
|
+
})
|
|
179
|
+
return r.data().count
|
|
180
|
+
}
|
|
181
|
+
|
|
148
182
|
const r = await firestoreQuery.count().get()
|
|
149
183
|
return r.data().count
|
|
150
184
|
}
|
|
@@ -162,7 +196,7 @@ export class FirestoreDB extends BaseCommonDB implements CommonDB {
|
|
|
162
196
|
...opt_,
|
|
163
197
|
}
|
|
164
198
|
|
|
165
|
-
if (opt.experimentalCursorStream) {
|
|
199
|
+
if (opt.experimentalCursorStream || opt.readAt) {
|
|
166
200
|
return Pipeline.from(new FirestoreStreamReadable(firestoreQuery, q, opt))
|
|
167
201
|
}
|
|
168
202
|
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { Readable } from 'node:stream'
|
|
2
|
-
import { FieldPath
|
|
2
|
+
import { FieldPath } from '@google-cloud/firestore'
|
|
3
|
+
import type { Query, QuerySnapshot } from '@google-cloud/firestore'
|
|
3
4
|
import type { DBQuery } from '@naturalcycles/db-lib'
|
|
4
5
|
import { localTime } from '@naturalcycles/js-lib/datetime'
|
|
5
6
|
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
|
|
6
|
-
import {
|
|
7
|
+
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
|
|
8
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
7
9
|
import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
|
|
8
10
|
import type {
|
|
9
11
|
ObjectWithId,
|
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
import { Readable } from 'node:stream'
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
import { FieldPath } from '@google-cloud/firestore'
|
|
3
|
+
import type {
|
|
4
|
+
Query,
|
|
5
|
+
QueryDocumentSnapshot,
|
|
6
|
+
QuerySnapshot,
|
|
7
|
+
Timestamp,
|
|
7
8
|
} from '@google-cloud/firestore'
|
|
8
9
|
import type { DBQuery } from '@naturalcycles/db-lib'
|
|
9
10
|
import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
|
|
10
11
|
import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
|
|
11
|
-
import {
|
|
12
|
+
import { TimeoutError } from '@naturalcycles/js-lib/error/error.util.js'
|
|
13
|
+
import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
|
|
14
|
+
import type { CommonLogger } from '@naturalcycles/js-lib/log'
|
|
12
15
|
import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
|
|
13
16
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types'
|
|
14
17
|
import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
|
|
15
18
|
import type { FirestoreDBStreamOptions } from './firestore.db.js'
|
|
16
19
|
import { unescapeDocId } from './firestore.util.js'
|
|
20
|
+
import { readAtToReadTime } from './query.util.js'
|
|
17
21
|
|
|
18
22
|
export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
19
23
|
extends Readable
|
|
@@ -32,6 +36,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
32
36
|
*/
|
|
33
37
|
countReads = 0
|
|
34
38
|
|
|
39
|
+
private readonly readTime?: Timestamp
|
|
35
40
|
private readonly opt: FirestoreDBStreamOptions & { batchSize: number; highWaterMark: number }
|
|
36
41
|
private logger: CommonLogger
|
|
37
42
|
|
|
@@ -51,7 +56,7 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
51
56
|
batchSize,
|
|
52
57
|
highWaterMark,
|
|
53
58
|
}
|
|
54
|
-
|
|
59
|
+
this.readTime = readAtToReadTime(opt)
|
|
55
60
|
|
|
56
61
|
this.originalLimit = dbQuery._limitValue
|
|
57
62
|
this.table = dbQuery.table
|
|
@@ -178,11 +183,19 @@ export class FirestoreStreamReadable<T extends ObjectWithId = any>
|
|
|
178
183
|
try {
|
|
179
184
|
return await pRetry(
|
|
180
185
|
async () => {
|
|
186
|
+
if (this.readTime) {
|
|
187
|
+
return await this.q.firestore.runTransaction(tx => tx.get(q), {
|
|
188
|
+
readOnly: true,
|
|
189
|
+
readTime: this.readTime,
|
|
190
|
+
})
|
|
191
|
+
}
|
|
181
192
|
return await q.get()
|
|
182
193
|
},
|
|
183
194
|
{
|
|
184
195
|
name: `FirestoreStreamReadable.query(${table})`,
|
|
185
|
-
predicate: err =>
|
|
196
|
+
predicate: err =>
|
|
197
|
+
err instanceof TimeoutError ||
|
|
198
|
+
RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
|
|
186
199
|
maxAttempts: 5,
|
|
187
200
|
delay: 5000,
|
|
188
201
|
delayMultiplier: 2,
|
package/src/query.util.ts
CHANGED
|
@@ -1,7 +1,21 @@
|
|
|
1
|
-
import { FieldPath,
|
|
2
|
-
import type {
|
|
1
|
+
import { FieldPath, Timestamp } from '@google-cloud/firestore'
|
|
2
|
+
import type { Query, WhereFilterOp } from '@google-cloud/firestore'
|
|
3
|
+
import type { CommonDBReadOptions, DBQuery, DBQueryFilterOperator } from '@naturalcycles/db-lib'
|
|
4
|
+
import { _round } from '@naturalcycles/js-lib'
|
|
3
5
|
import type { ObjectWithId } from '@naturalcycles/js-lib/types'
|
|
4
6
|
|
|
7
|
+
export function readAtToReadTime(opt: CommonDBReadOptions): Timestamp | undefined {
|
|
8
|
+
if (!opt.readAt) return
|
|
9
|
+
|
|
10
|
+
// Same logic as Datastore: round to whole minutes, guard against future
|
|
11
|
+
let readTimeMs = _round(opt.readAt, 60) * 1000
|
|
12
|
+
if (readTimeMs >= Date.now() - 1000) {
|
|
13
|
+
readTimeMs -= 60_000
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
return Timestamp.fromMillis(readTimeMs)
|
|
17
|
+
}
|
|
18
|
+
|
|
5
19
|
// Map DBQueryFilterOp to WhereFilterOp
|
|
6
20
|
// Currently it's fully aligned!
|
|
7
21
|
const OP_MAP: Partial<Record<DBQueryFilterOperator, WhereFilterOp>> = {
|