@naturalcycles/datastore-lib 4.18.3 → 4.18.5

package/dist/datastore.db.d.ts

@@ -66,7 +66,6 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
      * It may happen that transaction is already committed/rolled back, so we don't want to throw an error here.
      */
     private rollback;
-    private getRunQueryOptions;
 }
 /**
  * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs

package/dist/datastore.db.js

@@ -1,6 +1,5 @@
 import { Datastore, PropertyFilter } from '@google-cloud/datastore';
 import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
-import { _round } from '@naturalcycles/js-lib';
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
@@ -14,7 +13,7 @@ import { boldWhite } from '@naturalcycles/nodejs-lib/colors';
 import { Pipeline } from '@naturalcycles/nodejs-lib/stream';
 import { DatastoreType } from './datastore.model.js';
 import { DatastoreStreamReadable } from './datastoreStreamReadable.js';
-import { dbQueryToDatastoreQuery } from './query.util.js';
+import { dbQueryToDatastoreQuery, getRunQueryOptions } from './query.util.js';
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
 // It's an empyrical value, but anything less than infinity is better than infinity
@@ -90,7 +89,7 @@ export class DatastoreDB extends BaseCommonDB {
         let ds = this.ds();
         const keys = ids.map(id => this.key(ds, table, id));
         let rows;
-        const dsOpt = this.getRunQueryOptions(opt);
+        const dsOpt = getRunQueryOptions(opt);
         if (this.cfg.timeout) {
             // First try
             try {
@@ -141,7 +140,7 @@ export class DatastoreDB extends BaseCommonDB {
     async multiGet(map, opt = {}) {
         const result = {};
         const ds = this.ds();
-        const dsOpt = this.getRunQueryOptions(opt);
+        const dsOpt = getRunQueryOptions(opt);
         const keys = [];
         for (const [table, ids] of _stringMapEntries(map)) {
             result[table] = [];
@@ -174,7 +173,7 @@ export class DatastoreDB extends BaseCommonDB {
         }
         const ds = this.ds();
         const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
-        const dsOpt = this.getRunQueryOptions(opt);
+        const dsOpt = getRunQueryOptions(opt);
         const qr = await this.runDatastoreQuery(q, dsOpt);
         // Special case when projection query didn't specify 'id'
         if (dbQuery._selectedFieldNames && !dbQuery._selectedFieldNames.includes('id')) {
@@ -186,7 +185,7 @@ export class DatastoreDB extends BaseCommonDB {
         const ds = this.ds();
         const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table));
         const aq = ds.createAggregationQuery(q).count('count');
-        const dsOpt = this.getRunQueryOptions(opt);
+        const dsOpt = getRunQueryOptions(opt);
         const [entities] = await ds.runAggregationQuery(aq, dsOpt);
         return entities[0]?.count;
     }
@@ -209,7 +208,7 @@ export class DatastoreDB extends BaseCommonDB {
         };
         const readable = opt.experimentalCursorStream
             ? new DatastoreStreamReadable(q, opt)
-            : ds.runQueryStream(q, this.getRunQueryOptions(opt));
+            : ds.runQueryStream(q, getRunQueryOptions(opt));
         return Pipeline.from(readable).mapSync(r => this.mapId(r));
     }
     // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js
@@ -255,7 +254,7 @@ export class DatastoreDB extends BaseCommonDB {
         }
         const ds = this.ds();
         const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table));
-        const dsOpt = this.getRunQueryOptions(opt);
+        const dsOpt = getRunQueryOptions(opt);
         const { rows } = await this.runDatastoreQuery(datastoreQuery, dsOpt);
         return await this.deleteByIds(q.table, rows.map(obj => obj.id), opt);
     }
@@ -500,18 +499,6 @@ export class DatastoreDB extends BaseCommonDB {
             this.cfg.logger.error(err);
         }
     }
-    getRunQueryOptions(opt) {
-        if (!opt.readAt)
-            return {};
-        // Datastore expects UnixTimestamp in milliseconds
-        // Datastore requires the timestamp to be rounded to the whole minutes
-        const readTime = _round(opt.readAt, 60) * 1000;
-        if (readTime >= Date.now() - 1000) {
-            // To avoid the error of: The requested 'read_time' cannot be in the future
-            return {};
-        }
-        return { readTime };
-    }
 }
 /**
  * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs

package/dist/datastoreStreamReadable.js

@@ -3,6 +3,7 @@ import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
+import { getRunQueryOptions } from './query.util.js';
 export class DatastoreStreamReadable extends Readable {
     q;
     table;
@@ -39,11 +40,7 @@ export class DatastoreStreamReadable extends Readable {
             batchSize,
             highWaterMark,
         };
-        this.dsOpt = {};
-        if (opt.readAt) {
-            // Datastore expects UnixTimestamp in milliseconds
-            this.dsOpt.readTime = opt.readAt * 1000;
-        }
+        this.dsOpt = getRunQueryOptions(opt);
         const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
         this.logger = logger;
         this.originalLimit = q.limitVal;
@@ -82,7 +79,7 @@ export class DatastoreStreamReadable extends Readable {
             return;
         }
         if (this.paused) {
-            this.logger.log(`_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`);
+            this.logger.debug(`_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`);
             this.paused = false;
         }
         if (this.queryIsRunning) {
@@ -121,7 +118,7 @@ export class DatastoreStreamReadable extends Readable {
         const rows = res[0];
         const info = res[1];
         this.rowsRetrieved += rows.length;
-        logger.log(`${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
+        logger.debug(`${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
         this.endCursor = info.endCursor;
         this.queryIsRunning = false; // ready to take more _reads
         this.lastQueryDone = Date.now();
@@ -150,7 +147,7 @@ export class DatastoreStreamReadable extends Readable {
                 logger.debug(`${table} stream is already paused`);
             }
             else {
-                logger.log(`${table} pausing the stream`);
+                logger.debug(`${table} pausing the stream`);
                 this.paused = true;
             }
         }

package/dist/query.util.d.ts

@@ -1,4 +1,7 @@
 import { type Query } from '@google-cloud/datastore';
+import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query.js';
 import type { DBQuery } from '@naturalcycles/db-lib';
 import type { ObjectWithId } from '@naturalcycles/js-lib/types';
+import type { DatastoreDBReadOptions } from './datastore.model.js';
 export declare function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query): Query;
+export declare function getRunQueryOptions(opt: DatastoreDBReadOptions): RunQueryOptions;

package/dist/query.util.js

@@ -1,4 +1,5 @@
 import { PropertyFilter } from '@google-cloud/datastore';
+import { _round } from '@naturalcycles/js-lib';
 const FNAME_MAP = {
     id: '__key__',
 };
@@ -47,3 +48,15 @@ export function dbQueryToDatastoreQuery(dbQuery, emptyQuery) {
     }
     return q;
 }
+export function getRunQueryOptions(opt) {
+    if (!opt.readAt)
+        return {};
+    // Datastore expects UnixTimestamp in milliseconds
+    // Datastore requires the timestamp to be rounded to the whole minutes
+    let readTime = _round(opt.readAt, 60) * 1000;
+    if (readTime >= Date.now() - 1000) {
+        // To avoid the error of: The requested 'read_time' cannot be in the future
+        readTime -= 60_000;
+    }
+    return { readTime };
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/datastore-lib",
   "type": "module",
-  "version": "4.18.3",
+  "version": "4.18.5",
   "description": "Opinionated library to work with Google Datastore, implements CommonDB",
   "dependencies": {
     "@google-cloud/datastore": "^10",

package/src/datastore.db.ts

@@ -15,7 +15,6 @@ import type {
   RunQueryResult,
 } from '@naturalcycles/db-lib'
 import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib'
-import { _round } from '@naturalcycles/js-lib'
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js'
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
 import { _assert } from '@naturalcycles/js-lib/error/assert.js'
@@ -47,7 +46,7 @@ import type {
 } from './datastore.model.js'
 import { DatastoreType } from './datastore.model.js'
 import { DatastoreStreamReadable } from './datastoreStreamReadable.js'
-import { dbQueryToDatastoreQuery } from './query.util.js'
+import { dbQueryToDatastoreQuery, getRunQueryOptions } from './query.util.js'
 
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500
@@ -146,7 +145,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     const keys = ids.map(id => this.key(ds, table, id))
     let rows: any[]
 
-    const dsOpt = this.getRunQueryOptions(opt)
+    const dsOpt = getRunQueryOptions(opt)
 
     if (this.cfg.timeout) {
       // First try
@@ -216,7 +215,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   ): Promise<StringMap<ROW[]>> {
     const result: StringMap<ROW[]> = {}
     const ds = this.ds()
-    const dsOpt = this.getRunQueryOptions(opt)
+    const dsOpt = getRunQueryOptions(opt)
     const keys: Key[] = []
     for (const [table, ids] of _stringMapEntries(map)) {
       result[table] = []
@@ -260,7 +259,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
     const ds = this.ds()
     const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
-    const dsOpt = this.getRunQueryOptions(opt)
+    const dsOpt = getRunQueryOptions(opt)
     const qr = await this.runDatastoreQuery<ROW>(q, dsOpt)
 
     // Special case when projection query didn't specify 'id'
@@ -278,7 +277,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     const ds = this.ds()
     const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table))
     const aq = ds.createAggregationQuery(q).count('count')
-    const dsOpt = this.getRunQueryOptions(opt)
+    const dsOpt = getRunQueryOptions(opt)
     const [entities] = await ds.runAggregationQuery(aq, dsOpt)
     return entities[0]?.count
   }
@@ -313,7 +312,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
     const readable = opt.experimentalCursorStream
       ? new DatastoreStreamReadable<ROW>(q, opt)
-      : ds.runQueryStream(q, this.getRunQueryOptions(opt))
+      : ds.runQueryStream(q, getRunQueryOptions(opt))
 
     return Pipeline.from<ROW>(readable).mapSync(r => this.mapId<ROW>(r))
   }
@@ -383,7 +382,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
     const ds = this.ds()
     const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table))
-    const dsOpt = this.getRunQueryOptions(opt)
+    const dsOpt = getRunQueryOptions(opt)
     const { rows } = await this.runDatastoreQuery<ObjectWithId>(datastoreQuery, dsOpt)
     return await this.deleteByIds(
       q.table,
@@ -680,20 +679,6 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       this.cfg.logger.error(err)
     }
   }
-
-  private getRunQueryOptions(opt: DatastoreDBReadOptions): RunQueryOptions {
-    if (!opt.readAt) return {}
-
-    // Datastore expects UnixTimestamp in milliseconds
-    // Datastore requires the timestamp to be rounded to the whole minutes
-    const readTime = _round(opt.readAt, 60) * 1000
-    if (readTime >= Date.now() - 1000) {
-      // To avoid the error of: The requested 'read_time' cannot be in the future
-      return {}
-    }
-
-    return { readTime }
-  }
 }
 
 /**

package/src/datastoreStreamReadable.ts

@@ -12,6 +12,7 @@ import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
 import type { UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import type { DatastoreDBStreamOptions } from './datastore.model.js'
+import { getRunQueryOptions } from './query.util.js'
 
 export class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
   private readonly table: string
@@ -53,12 +54,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       batchSize,
       highWaterMark,
     }
-    this.dsOpt = {}
-    if (opt.readAt) {
-      // Datastore expects UnixTimestamp in milliseconds
-      this.dsOpt.readTime = opt.readAt * 1000
-    }
-
+    this.dsOpt = getRunQueryOptions(opt)
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
     this.logger = logger
     this.originalLimit = q.limitVal
@@ -112,7 +108,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     }
 
     if (this.paused) {
-      this.logger.log(
+      this.logger.debug(
         `_read #${this.countReads}, queryIsRunning: ${this.queryIsRunning}, unpausing stream`,
       )
       this.paused = false
@@ -162,7 +158,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     const info: RunQueryInfo = res[1]
 
     this.rowsRetrieved += rows.length
-    logger.log(
+    logger.debug(
       `${table} got ${rows.length} rows in ${_ms(queryTook)}, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(
         this.totalWait,
       )}`,
@@ -201,7 +197,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       if (this.paused) {
         logger.debug(`${table} stream is already paused`)
       } else {
-        logger.log(`${table} pausing the stream`)
+        logger.debug(`${table} pausing the stream`)
         this.paused = true
       }
     }

package/src/query.util.ts

@@ -1,6 +1,9 @@
 import { PropertyFilter, type Query } from '@google-cloud/datastore'
+import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
 import type { DBQuery, DBQueryFilterOperator } from '@naturalcycles/db-lib'
+import { _round } from '@naturalcycles/js-lib'
 import type { ObjectWithId, StringMap } from '@naturalcycles/js-lib/types'
+import type { DatastoreDBReadOptions } from './datastore.model.js'
 
 const FNAME_MAP: StringMap = {
   id: '__key__',
@@ -64,3 +67,17 @@ export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
 
   return q
 }
+
+export function getRunQueryOptions(opt: DatastoreDBReadOptions): RunQueryOptions {
+  if (!opt.readAt) return {}
+
+  // Datastore expects UnixTimestamp in milliseconds
+  // Datastore requires the timestamp to be rounded to the whole minutes
+  let readTime = _round(opt.readAt, 60) * 1000
+  if (readTime >= Date.now() - 1000) {
+    // To avoid the error of: The requested 'read_time' cannot be in the future
+    readTime -= 60_000
+  }
+
+  return { readTime }
+}