@naturalcycles/datastore-lib 3.14.1 → 3.16.2

package/CHANGELOG.md

@@ -1,3 +1,31 @@
+## [3.16.2](https://github.com/NaturalCycles/datastore-lib/compare/v3.16.1...v3.16.2) (2021-10-17)
+
+
+### Bug Fixes
+
+* adapt to latest db-lib with typed queries ([21d31ea](https://github.com/NaturalCycles/datastore-lib/commit/21d31ea7343ca0b2590950c067f6ce472dc82501))
+
+## [3.16.1](https://github.com/NaturalCycles/datastore-lib/compare/v3.16.0...v3.16.1) (2021-10-07)
+
+
+### Bug Fixes
+
+* logging ([961b485](https://github.com/NaturalCycles/datastore-lib/commit/961b485a0ff1c0268bfc491e9f7a608a1b2b77da))
+
+# [3.16.0](https://github.com/NaturalCycles/datastore-lib/compare/v3.15.0...v3.16.0) (2021-10-07)
+
+
+### Features
+
+* allow to set streamOptions in DatastoreCfg ([c9fb5f0](https://github.com/NaturalCycles/datastore-lib/commit/c9fb5f08760f113d3b563ce80579df189d06b7b1))
+
+# [3.15.0](https://github.com/NaturalCycles/datastore-lib/compare/v3.14.1...v3.15.0) (2021-10-07)
+
+
+### Features
+
+* experimentalCursorStream mode to fix backpressure ([419f8e1](https://github.com/NaturalCycles/datastore-lib/commit/419f8e16ccb9722771a01b90001ae19f2a7a8595))
+
 ## [3.14.1](https://github.com/NaturalCycles/datastore-lib/compare/v3.14.0...v3.14.1) (2021-10-04)
 
 

package/dist/DatastoreStreamReadable.d.ts

@@ -0,0 +1,21 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { Query } from '@google-cloud/datastore';
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import type { DatastoreDBStreamOptions } from './datastore.model';
+export declare class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
+    private q;
+    private originalLimit;
+    private rowsRetrieved;
+    private endCursor?;
+    private running;
+    private done;
+    private lastQueryDone?;
+    private totalWait;
+    private log;
+    private opt;
+    constructor(q: Query, opt: DatastoreDBStreamOptions);
+    private runNextQuery;
+    count: number;
+    _read(): void;
+}

package/dist/DatastoreStreamReadable.js

@@ -0,0 +1,90 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DatastoreStreamReadable = void 0;
+const stream_1 = require("stream");
+const js_lib_1 = require("@naturalcycles/js-lib");
+class DatastoreStreamReadable extends stream_1.Readable {
+    constructor(q, opt) {
+        super({ objectMode: true });
+        this.q = q;
+        this.rowsRetrieved = 0;
+        this.running = false;
+        this.done = false;
+        this.totalWait = 0;
+        // private log = (...args: any[]): void => console.log(...args)
+        this.log = console.log.bind(console);
+        this.count = 0; // use for debugging
+        this.opt = {
+            rssLimitMB: 1000,
+            batchSize: 1000,
+            ...opt,
+        };
+        if (!opt.debug)
+            this.log = () => { };
+        this.originalLimit = q.limitVal;
+        console.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`);
+    }
+    async runNextQuery() {
+        if (this.done)
+            return;
+        if (this.lastQueryDone) {
+            const now = Date.now();
+            this.totalWait += now - this.lastQueryDone;
+        }
+        this.running = true;
+        // console.log('running query...')
+        let limit = this.opt.batchSize;
+        if (this.originalLimit) {
+            limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved);
+        }
+        // console.log(`limit: ${limit}`)
+        let q = this.q.limit(limit);
+        if (this.endCursor) {
+            q = q.start(this.endCursor);
+        }
+        try {
+            const [rows, info] = await q.run();
+            this.rowsRetrieved += rows.length;
+            this.log(`got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
+            this.endCursor = info.endCursor;
+            this.running = false; // ready to take more _reads
+            this.lastQueryDone = Date.now();
+            rows.forEach(row => this.push(row));
+            if (!info.endCursor ||
+                info.moreResults === 'NO_MORE_RESULTS' ||
+                (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
+                this.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
+                this.push(null);
+                this.done = true;
+            }
+            else if (this.opt.rssLimitMB) {
+                const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024);
+                if (rssMB <= this.opt.rssLimitMB) {
+                    void this.runNextQuery();
+                }
+                else {
+                    this.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`);
+                }
+            }
+        }
+        catch (err) {
+            console.error('DatastoreStreamReadable error!\n', err);
+            this.emit('error', err);
+        }
+    }
+    _read() {
+        // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+        this.count++;
+        if (this.running) {
+            this.log(`_read ${this.count}, wasRunning: true`);
+        }
+        if (this.done) {
+            console.warn(`!!! _read was called, but done==true`);
+            return;
+        }
+        if (!this.running) {
+            void this.runNextQuery();
+        }
+    }
+}
+exports.DatastoreStreamReadable = DatastoreStreamReadable;

package/dist/datastore.db.d.ts

@@ -2,7 +2,7 @@ import type { Datastore, Key, Query } from '@google-cloud/datastore';
 import { BaseCommonDB, CommonDB, DBQuery, DBTransaction, ObjectWithId, RunQueryResult } from '@naturalcycles/db-lib';
 import { JsonSchemaRootObject } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats } from './datastore.model';
+import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats } from './datastore.model';
 /**
  * Datastore API:
  * https://googlecloudplatform.github.io/google-cloud-node/#/docs/datastore/1.0.3/datastore
@@ -23,12 +23,12 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<number>;
     runDatastoreQuery<ROW extends ObjectWithId>(q: Query): Promise<RunQueryResult<ROW>>;
-    runQueryStream<ROW extends ObjectWithId>(q: Query): ReadableTyped<ROW>;
-    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): ReadableTyped<ROW>;
+    private runQueryStream;
+    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBStreamOptions): ReadableTyped<ROW>;
     /**
      * Returns saved entities with generated id/updated/created (non-mutating!)
      */
-    saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: DatastoreDBSaveOptions): Promise<void>;
+    saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: DatastoreDBSaveOptions<ROW>): Promise<void>;
     deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: DatastoreDBOptions): Promise<number>;
     /**
      * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"

package/dist/datastore.db.js

@@ -6,6 +6,7 @@ const db_lib_1 = require("@naturalcycles/db-lib");
 const js_lib_1 = require("@naturalcycles/js-lib");
 const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
 const datastore_model_1 = require("./datastore.model");
+const DatastoreStreamReadable_1 = require("./DatastoreStreamReadable");
 const query_util_1 = require("./query.util");
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
@@ -88,21 +89,23 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             rows,
         };
     }
-    runQueryStream(q) {
-        return (this.ds()
-            .runQueryStream(q)
-            // Important that new instance of Transform should be created every time!
-            // Otherwise they share shate and affect each other
-            .pipe(new stream_1.Transform({
+    runQueryStream(q, _opt) {
+        const opt = {
+            ...this.cfg.streamOptions,
+            ..._opt,
+        };
+        return (opt.experimentalCursorStream
+            ? new DatastoreStreamReadable_1.DatastoreStreamReadable(q, opt)
+            : this.ds().runQueryStream(q)).pipe(new stream_1.Transform({
             objectMode: true,
             transform: (chunk, _enc, cb) => {
                 cb(null, this.mapId(chunk));
             },
-        })));
+        }));
     }
-    streamQuery(dbQuery, _opt) {
+    streamQuery(dbQuery, opt) {
         const q = (0, query_util_1.dbQueryToDatastoreQuery)(dbQuery, this.ds().createQuery(dbQuery.table));
-        return this.runQueryStream(q);
+        return this.runQueryStream(q, opt);
     }
     // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js
     /**

package/dist/datastore.model.d.ts

@@ -1,5 +1,6 @@
 import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore';
 import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib';
+import { AnyObjectWithId, ObjectWithId } from '@naturalcycles/db-lib/src/db.model';
 export interface DatastorePayload<T = any> {
     key: Key;
     data: T;
@@ -24,6 +25,11 @@ export interface DatastoreDBCfg extends DatastoreOptions {
      * Allows to set the old native library here, e.g `grpc: require('grpc')`
      */
     grpc?: any;
+    /**
+     * Use it to set default options to stream operations,
+     * e.g you can globally enable `experimentalCursorStream` here, set the batchSize, etc.
+     */
+    streamOptions?: DatastoreDBStreamOptions;
 }
 export interface DatastoreCredentials {
     type?: string;
@@ -35,10 +41,43 @@ export interface DatastoreCredentials {
     client_secret?: string;
     refresh_token?: string;
 }
+export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
+    /**
+     * Set to `true` to stream via experimental "cursor-query based stream".
+     *
+     * @default false
+     */
+    experimentalCursorStream?: boolean;
+    /**
+     * Applicable to `experimentalCursorStream`
+     *
+     * @default 1000
+     */
+    batchSize?: number;
+    /**
+     * Applicable to `experimentalCursorStream`
+     *
+     * Set to a value (number of Megabytes) to control the peak RSS size.
+     * If limit is reached - streaming will pause until the stream keeps up, and then
+     * resumes.
+     *
+     * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
+     * when _read is called.
+     *
+     * @default 1000
+     */
+    rssLimitMB?: number;
+    /**
+     * Set to `true` to log additional debug info, when using experimentalCursorStream.
+     *
+     * @default false
+     */
+    debug?: boolean;
+}
 export interface DatastoreDBOptions extends CommonDBOptions {
     tx?: Transaction;
 }
-export interface DatastoreDBSaveOptions extends CommonDBSaveOptions {
+export interface DatastoreDBSaveOptions<ROW extends ObjectWithId = AnyObjectWithId> extends CommonDBSaveOptions<ROW> {
     tx?: Transaction;
 }
 export interface DatastoreStats {

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { DatastoreDB } from './datastore.db';
-import { DatastoreCredentials, DatastoreDBCfg, DatastorePayload, DatastorePropertyStats, DatastoreStats, DatastoreType } from './datastore.model';
+import { DatastoreCredentials, DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats, DatastoreType } from './datastore.model';
 import { DatastoreKeyValueDB, DatastoreKeyValueDBCfg } from './datastoreKeyValueDB';
 import { getDBAdapter } from './dbAdapter';
-export type { DatastorePayload, DatastoreDBCfg, DatastoreKeyValueDBCfg, DatastoreStats, DatastoreCredentials, DatastorePropertyStats, };
+export type { DatastorePayload, DatastoreDBCfg, DatastoreKeyValueDBCfg, DatastoreStats, DatastoreCredentials, DatastorePropertyStats, DatastoreDBStreamOptions, DatastoreDBOptions, DatastoreDBSaveOptions, };
 export { DatastoreDB, DatastoreType, getDBAdapter, DatastoreKeyValueDB };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.14.1",
+  "version": "3.16.2",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"

package/src/DatastoreStreamReadable.ts

@@ -0,0 +1,120 @@
+import { Readable } from 'stream'
+import { Query } from '@google-cloud/datastore'
+import { _ms } from '@naturalcycles/js-lib'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import type { DatastoreDBStreamOptions } from './datastore.model'
+
+export class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
+  private originalLimit: number
+  private rowsRetrieved = 0
+  private endCursor?: string
+  private running = false
+  private done = false
+  private lastQueryDone?: number
+  private totalWait = 0
+
+  // private log = (...args: any[]): void => console.log(...args)
+  private log = console.log.bind(console)
+
+  private opt: DatastoreDBStreamOptions & { batchSize: number }
+
+  constructor(private q: Query, opt: DatastoreDBStreamOptions) {
+    super({ objectMode: true })
+
+    this.opt = {
+      rssLimitMB: 1000,
+      batchSize: 1000,
+      ...opt,
+    }
+
+    if (!opt.debug) this.log = () => {}
+
+    this.originalLimit = q.limitVal
+
+    console.log(`!! using experimentalCursorStream !! batchSize: ${opt.batchSize}`)
+  }
+
+  private async runNextQuery(): Promise<void> {
+    if (this.done) return
+
+    if (this.lastQueryDone) {
+      const now = Date.now()
+      this.totalWait += now - this.lastQueryDone
+    }
+
+    this.running = true
+    // console.log('running query...')
+
+    let limit = this.opt.batchSize
+
+    if (this.originalLimit) {
+      limit = Math.min(this.opt.batchSize, this.originalLimit - this.rowsRetrieved)
+    }
+
+    // console.log(`limit: ${limit}`)
+    let q = this.q.limit(limit)
+    if (this.endCursor) {
+      q = q.start(this.endCursor)
+    }
+
+    try {
+      const [rows, info] = await q.run()
+
+      this.rowsRetrieved += rows.length
+      this.log(
+        `got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
+          this.totalWait,
+        )}`,
+        info.moreResults,
+      )
+
+      this.endCursor = info.endCursor
+      this.running = false // ready to take more _reads
+      this.lastQueryDone = Date.now()
+
+      rows.forEach(row => this.push(row))
+
+      if (
+        !info.endCursor ||
+        info.moreResults === 'NO_MORE_RESULTS' ||
+        (this.originalLimit && this.rowsRetrieved >= this.originalLimit)
+      ) {
+        this.log(
+          `!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`,
+        )
+        this.push(null)
+        this.done = true
+      } else if (this.opt.rssLimitMB) {
+        const rssMB = Math.round(process.memoryUsage().rss / 1024 / 1024)
+
+        if (rssMB <= this.opt.rssLimitMB) {
+          void this.runNextQuery()
+        } else {
+          this.log(`rssLimitMB reached ${rssMB} > ${this.opt.rssLimitMB}, pausing stream`)
+        }
+      }
+    } catch (err) {
+      console.error('DatastoreStreamReadable error!\n', err)
+      this.emit('error', err)
+    }
+  }
+
+  count = 0 // use for debugging
+
+  override _read(): void {
+    // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
+    this.count++
+    if (this.running) {
+      this.log(`_read ${this.count}, wasRunning: true`)
+    }
+
+    if (this.done) {
+      console.warn(`!!! _read was called, but done==true`)
+      return
+    }
+
+    if (!this.running) {
+      void this.runNextQuery()
+    }
+  }
+}

package/src/datastore.db.ts

@@ -29,11 +29,13 @@ import {
   DatastoreDBCfg,
   DatastoreDBOptions,
   DatastoreDBSaveOptions,
+  DatastoreDBStreamOptions,
   DatastorePayload,
   DatastorePropertyStats,
   DatastoreStats,
   DatastoreType,
 } from './datastore.model'
+import { DatastoreStreamReadable } from './DatastoreStreamReadable'
 import { dbQueryToDatastoreQuery } from './query.util'
 
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
@@ -154,29 +156,35 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  runQueryStream<ROW extends ObjectWithId>(q: Query): ReadableTyped<ROW> {
+  private runQueryStream<ROW extends ObjectWithId>(
+    q: Query,
+    _opt?: DatastoreDBStreamOptions,
+  ): ReadableTyped<ROW> {
+    const opt = {
+      ...this.cfg.streamOptions,
+      ..._opt,
+    }
+
     return (
-      this.ds()
-        .runQueryStream(q)
-        // Important that new instance of Transform should be created every time!
-        // Otherwise they share shate and affect each other
-        .pipe(
-          new Transform({
-            objectMode: true,
-            transform: (chunk, _enc, cb) => {
-              cb(null, this.mapId(chunk))
-            },
-          }),
-        )
+      opt.experimentalCursorStream
+        ? new DatastoreStreamReadable(q, opt)
+        : this.ds().runQueryStream(q)
+    ).pipe(
+      new Transform({
+        objectMode: true,
+        transform: (chunk, _enc, cb) => {
+          cb(null, this.mapId(chunk))
+        },
+      }),
     )
   }
 
   override streamQuery<ROW extends ObjectWithId>(
     dbQuery: DBQuery<ROW>,
-    _opt?: DatastoreDBOptions,
+    opt?: DatastoreDBStreamOptions,
   ): ReadableTyped<ROW> {
     const q = dbQueryToDatastoreQuery(dbQuery, this.ds().createQuery(dbQuery.table))
-    return this.runQueryStream(q)
+    return this.runQueryStream(q, opt)
   }
 
   // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js
@@ -187,9 +195,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   override async saveBatch<ROW extends ObjectWithId>(
     table: string,
     rows: ROW[],
-    opt: DatastoreDBSaveOptions = {},
+    opt: DatastoreDBSaveOptions<ROW> = {},
   ): Promise<void> {
-    const entities = rows.map(obj => this.toDatastoreEntity(table, obj, opt.excludeFromIndexes))
+    const entities = rows.map(obj =>
+      this.toDatastoreEntity(table, obj, opt.excludeFromIndexes as string[]),
+    )
 
     const save = pRetry(
       async (batch: DatastorePayload<ROW>[]) => {

package/src/datastore.model.ts

@@ -1,5 +1,6 @@
 import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore'
 import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib'
+import { AnyObjectWithId, ObjectWithId } from '@naturalcycles/db-lib/src/db.model'
 
 export interface DatastorePayload<T = any> {
   key: Key
@@ -29,6 +30,12 @@ export interface DatastoreDBCfg extends DatastoreOptions {
    * Allows to set the old native library here, e.g `grpc: require('grpc')`
    */
   grpc?: any
+
+  /**
+   * Use it to set default options to stream operations,
+   * e.g you can globally enable `experimentalCursorStream` here, set the batchSize, etc.
+   */
+  streamOptions?: DatastoreDBStreamOptions
 }
 
 export interface DatastoreCredentials {
@@ -42,10 +49,48 @@ export interface DatastoreCredentials {
   refresh_token?: string
 }
 
+export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
+  /**
+   * Set to `true` to stream via experimental "cursor-query based stream".
+   *
+   * @default false
+   */
+  experimentalCursorStream?: boolean
+
+  /**
+   * Applicable to `experimentalCursorStream`
+   *
+   * @default 1000
+   */
+  batchSize?: number
+
+  /**
+   * Applicable to `experimentalCursorStream`
+   *
+   * Set to a value (number of Megabytes) to control the peak RSS size.
+   * If limit is reached - streaming will pause until the stream keeps up, and then
+   * resumes.
+   *
+   * Set to 0/undefined to disable. Stream will get "slow" then, cause it'll only run the query
+   * when _read is called.
+   *
+   * @default 1000
+   */
+  rssLimitMB?: number
+
+  /**
+   * Set to `true` to log additional debug info, when using experimentalCursorStream.
+   *
+   * @default false
+   */
+  debug?: boolean
+}
+
 export interface DatastoreDBOptions extends CommonDBOptions {
   tx?: Transaction
 }
-export interface DatastoreDBSaveOptions extends CommonDBSaveOptions {
+export interface DatastoreDBSaveOptions<ROW extends ObjectWithId = AnyObjectWithId>
+  extends CommonDBSaveOptions<ROW> {
   tx?: Transaction
 }
 

package/src/datastoreKeyValueDB.ts

@@ -9,7 +9,7 @@ interface KVObject {
   v: Buffer
 }
 
-const excludeFromIndexes = ['v']
+const excludeFromIndexes: (keyof KVObject)[] = ['v']
 
 export interface DatastoreKeyValueDBCfg extends DatastoreDBCfg {}
 

package/src/index.ts

@@ -2,6 +2,9 @@ import { DatastoreDB } from './datastore.db'
 import {
   DatastoreCredentials,
   DatastoreDBCfg,
+  DatastoreDBOptions,
+  DatastoreDBSaveOptions,
+  DatastoreDBStreamOptions,
   DatastorePayload,
   DatastorePropertyStats,
   DatastoreStats,
@@ -17,6 +20,9 @@ export type {
   DatastoreStats,
   DatastoreCredentials,
   DatastorePropertyStats,
+  DatastoreDBStreamOptions,
+  DatastoreDBOptions,
+  DatastoreDBSaveOptions,
 }
 
 export { DatastoreDB, DatastoreType, getDBAdapter, DatastoreKeyValueDB }

package/src/query.util.ts

@@ -19,18 +19,24 @@ export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
 
   // filter
   // eslint-disable-next-line unicorn/no-array-reduce
-  q = dbQuery._filters.reduce((q, f) => q.filter(f.name, OP_MAP[f.op] || (f.op as any), f.val), q)
+  q = dbQuery._filters.reduce(
+    (q, f) => q.filter(f.name as string, OP_MAP[f.op] || (f.op as any), f.val),
+    q,
+  )
 
   // limit
   q = q.limit(dbQuery._limitValue || 0)
 
   // order
   // eslint-disable-next-line unicorn/no-array-reduce
-  q = dbQuery._orders.reduce((q, ord) => q.order(ord.name, { descending: ord.descending }), q)
+  q = dbQuery._orders.reduce(
+    (q, ord) => q.order(ord.name as string, { descending: ord.descending }),
+    q,
+  )
 
   // select
   if (dbQuery._selectedFieldNames) {
-    const fields = dbQuery._selectedFieldNames.map(f => FNAME_MAP[f] || f)
+    const fields = (dbQuery._selectedFieldNames as string[]).map(f => FNAME_MAP[f] || f)
 
     // Datastore requires you to specify at least one column, so if empty array is passed - it'll include __key__ at least
     if (!fields.length) {