@naturalcycles/datastore-lib 4.14.0 → 4.15.1

package/dist/datastore.db.js

@@ -4,7 +4,6 @@ import { _round } from '@naturalcycles/js-lib';
 import { _chunk } from '@naturalcycles/js-lib/array/array.util.js';
 import { _assert } from '@naturalcycles/js-lib/error/assert.js';
 import { _errorDataAppend, TimeoutError } from '@naturalcycles/js-lib/error/error.util.js';
-import { commonLoggerMinLevel } from '@naturalcycles/js-lib/log';
 import { _omit } from '@naturalcycles/js-lib/object/object.util.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { pRetry, pRetryFn } from '@naturalcycles/js-lib/promise/pRetry.js';
@@ -219,13 +218,14 @@ export class DatastoreDB extends BaseCommonDB {
         void this.ds().then(async (ds) => {
             const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table), await this.getPropertyFilter());
             const opt = {
+                logger: this.cfg.logger,
                 ...this.cfg.streamOptions,
                 ..._opt,
             };
             (opt.experimentalCursorStream
-                ? new DatastoreStreamReadable(q, opt, commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'))
+                ? new DatastoreStreamReadable(q, opt)
                 : ds.runQueryStream(q, this.getRunQueryOptions(opt)))
-                .on('error', err => transform.emit('error', err))
+                .on('error', err => transform.destroy(err))
                 .pipe(transform);
         });
         return transform;

package/dist/datastore.model.d.ts

@@ -1,6 +1,6 @@
 import type { DatastoreOptions, Key } from '@google-cloud/datastore';
 import type { CommonDBOptions, CommonDBReadOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib';
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
+import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log';
 import type { NumberOfSeconds, ObjectWithId, PositiveInteger } from '@naturalcycles/js-lib/types';
 export interface DatastorePayload<T = any> {
     key: Key;
@@ -30,6 +30,10 @@ export interface DatastoreDBCfg extends DatastoreOptions {
      * Default to `console`
      */
     logger?: CommonLogger;
+    /**
+     * Defaults to `log`.
+     */
+    logLevel?: CommonLogLevel;
     /**
      * Experimental option, currently only applies to `getByIds`.
      * Applies pTimeout to Datastore operation, re-creates Datastore on any error.
@@ -70,12 +74,12 @@ export interface DatastoreDBStreamOptions extends DatastoreDBReadOptions {
      * between the queries.
      */
     highWaterMark?: PositiveInteger;
+    logger?: CommonLogger;
     /**
-     * Set to `true` to log additional debug info, when using experimentalCursorStream.
-     *
-     * @default false
+     * Defaults to `log`.
+     * Set to `debug` to allow for extra debugging, e.g in experimentalCursorStream.
      */
-    debug?: boolean;
+    logLevel?: CommonLogLevel;
     /**
      * Default is undefined.
      * If set - sets a "safety timer", which will force call _read after the specified number of seconds.

package/dist/datastoreStreamReadable.d.ts

@@ -1,11 +1,9 @@
 import { Readable } from 'node:stream';
 import type { Query } from '@google-cloud/datastore';
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
 import type { DatastoreDBStreamOptions } from './datastore.model.js';
 export declare class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
     private q;
-    private logger;
     private readonly table;
     private readonly originalLimit;
     private rowsRetrieved;
@@ -26,8 +24,9 @@ export declare class DatastoreStreamReadable<T = any> extends Readable implement
     private lastReadTimestamp;
     private readonly maxWaitInterval;
     private readonly opt;
+    private readonly logger;
     private readonly dsOpt;
-    constructor(q: Query, opt: DatastoreDBStreamOptions, logger: CommonLogger);
+    constructor(q: Query, opt: DatastoreDBStreamOptions);
     _read(): void;
     private runNextQuery;
     private runQuery;

package/dist/datastoreStreamReadable.js

@@ -1,10 +1,10 @@
 import { Readable } from 'node:stream';
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js';
 export class DatastoreStreamReadable extends Readable {
     q;
-    logger;
     table;
     originalLimit;
     rowsRetrieved = 0;
@@ -25,15 +25,15 @@ export class DatastoreStreamReadable extends Readable {
     lastReadTimestamp = 0;
     maxWaitInterval;
     opt;
+    logger;
     dsOpt;
-    constructor(q, opt, logger) {
+    constructor(q, opt) {
         // 1_000 was optimal in benchmarks
         const { batchSize = 1000 } = opt;
         const { highWaterMark = batchSize * 3 } = opt;
         // Defaulting highWaterMark to 3x batchSize
         super({ objectMode: true, highWaterMark });
         this.q = q;
-        this.logger = logger;
         this.opt = {
             ...opt,
             batchSize,
@@ -44,24 +44,26 @@ export class DatastoreStreamReadable extends Readable {
             // Datastore expects UnixTimestamp in milliseconds
             this.dsOpt.readTime = opt.readAt * 1000;
         }
+        const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+        this.logger = logger;
         this.originalLimit = q.limitVal;
         this.table = q.kinds[0];
-        logger.warn(`!! using experimentalCursorStream`, {
+        logger.log(`!! using experimentalCursorStream`, {
             table: this.table,
             batchSize,
             highWaterMark,
         });
         const { maxWait } = this.opt;
         if (maxWait) {
-            logger.warn(`!! ${this.table} maxWait ${maxWait}`);
+            logger.log(`!! ${this.table} maxWait ${maxWait}`);
             this.maxWaitInterval = setInterval(() => {
                 const millisSinceLastRead = Date.now() - this.lastReadTimestamp;
                 if (millisSinceLastRead < maxWait * 1000) {
-                    logger.warn(`!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`);
+                    logger.log(`!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`);
                     return;
                 }
                 const { queryIsRunning, rowsRetrieved } = this;
-                logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
+                logger.log(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
                     running: queryIsRunning,
                     rowsRetrieved,
                 });
@@ -72,7 +74,7 @@ export class DatastoreStreamReadable extends Readable {
         }
     }
     _read() {
-        this.lastReadTimestamp = Date.now();
+        this.lastReadTimestamp = localTime.nowUnixMillis();
         // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
         this.countReads++;
         if (this.done) {
@@ -84,12 +86,12 @@ export class DatastoreStreamReadable extends Readable {
             this.paused = false;
         }
         if (this.queryIsRunning) {
-            this.logger.log(`_read #${this.countReads}, queryIsRunning: true, doing nothing`);
+            this.logger.debug(`_read #${this.countReads}, queryIsRunning: true, doing nothing`);
             return;
         }
         void this.runNextQuery().catch(err => {
-            console.log('error in runNextQuery', err);
-            this.emit('error', err);
+            this.logger.error('error in runNextQuery', err);
+            this.destroy(err);
         });
     }
     async runNextQuery() {
@@ -139,13 +141,13 @@ export class DatastoreStreamReadable extends Readable {
         }
         if (shouldContinue) {
             // Keep the stream flowing
-            logger.log(`${table} continuing the stream`);
+            logger.debug(`${table} continuing the stream`);
             void this.runNextQuery();
         }
         else {
             // Not starting the next query
             if (this.paused) {
-                logger.log(`${table} stream is already paused`);
+                logger.debug(`${table} stream is already paused`);
             }
             else {
                 logger.log(`${table} pausing the stream`);
@@ -160,6 +162,7 @@ export class DatastoreStreamReadable extends Readable {
                 return await q.run(this.dsOpt);
             }, {
                 name: `DatastoreStreamReadable.query(${table})`,
+                predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
                 maxAttempts: 5,
                 delay: 5000,
                 delayMultiplier: 2,
@@ -168,13 +171,25 @@ export class DatastoreStreamReadable extends Readable {
             });
         }
         catch (err) {
-            console.log(`DatastoreStreamReadable error!\n`, {
+            logger.error(`DatastoreStreamReadable error!\n`, {
                 table,
                 rowsRetrieved: this.rowsRetrieved,
             }, err);
-            this.emit('error', err);
             clearInterval(this.maxWaitInterval);
+            this.destroy(err);
             return;
         }
     }
 }
+// Examples of errors:
+// UNKNOWN: Stream removed
+const RETRY_ON = [
+    'GOAWAY',
+    'UNAVAILABLE',
+    'UNKNOWN',
+    'DEADLINE_EXCEEDED',
+    'ABORTED',
+    'much contention',
+    'try again',
+    'timeout',
+].map(s => s.toLowerCase());

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/datastore-lib",
   "type": "module",
-  "version": "4.14.0",
+  "version": "4.15.1",
   "description": "Opinionated library to work with Google Datastore, implements CommonDB",
   "dependencies": {
     "@google-cloud/datastore": "^10",

package/src/datastore.db.ts

@@ -29,7 +29,6 @@ import type {
   JsonSchemaString,
 } from '@naturalcycles/js-lib/json-schema'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { commonLoggerMinLevel } from '@naturalcycles/js-lib/log'
 import { _omit } from '@naturalcycles/js-lib/object/object.util.js'
 import type { PRetryOptions } from '@naturalcycles/js-lib/promise'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
@@ -344,19 +343,16 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       )
 
       const opt = {
+        logger: this.cfg.logger,
         ...this.cfg.streamOptions,
         ..._opt,
       }
 
       ;(opt.experimentalCursorStream
-        ? new DatastoreStreamReadable<ROW>(
-            q,
-            opt,
-            commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
-          )
+        ? new DatastoreStreamReadable<ROW>(q, opt)
         : (ds.runQueryStream(q, this.getRunQueryOptions(opt)) as ReadableTyped<ROW>)
       )
-        .on('error', err => transform.emit('error', err))
+        .on('error', err => transform.destroy(err))
         .pipe(transform)
     })
 

package/src/datastore.model.ts

@@ -4,7 +4,7 @@ import type {
   CommonDBReadOptions,
   CommonDBSaveOptions,
 } from '@naturalcycles/db-lib'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
+import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log'
 import type { NumberOfSeconds, ObjectWithId, PositiveInteger } from '@naturalcycles/js-lib/types'
 
 export interface DatastorePayload<T = any> {
@@ -41,6 +41,11 @@ export interface DatastoreDBCfg extends DatastoreOptions {
    */
   logger?: CommonLogger
 
+  /**
+   * Defaults to `log`.
+   */
+  logLevel?: CommonLogLevel
+
   /**
    * Experimental option, currently only applies to `getByIds`.
    * Applies pTimeout to Datastore operation, re-creates Datastore on any error.
@@ -86,12 +91,13 @@ export interface DatastoreDBStreamOptions extends DatastoreDBReadOptions {
    */
   highWaterMark?: PositiveInteger
 
+  logger?: CommonLogger
+
   /**
-   * Set to `true` to log additional debug info, when using experimentalCursorStream.
-   *
-   * @default false
+   * Defaults to `log`.
+   * Set to `debug` to allow for extra debugging, e.g in experimentalCursorStream.
    */
-  debug?: boolean
+  logLevel?: CommonLogLevel
 
   /**
    * Default is undefined.

package/src/datastoreStreamReadable.ts

@@ -7,7 +7,7 @@ import type {
 } from '@google-cloud/datastore/build/src/query.js'
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js'
 import { _ms } from '@naturalcycles/js-lib/datetime/time.util.js'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
+import { type CommonLogger, createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import { pRetry } from '@naturalcycles/js-lib/promise/pRetry.js'
 import type { UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
@@ -35,12 +35,12 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
   private readonly maxWaitInterval: NodeJS.Timeout | undefined
 
   private readonly opt: DatastoreDBStreamOptions & { batchSize: number; highWaterMark: number }
+  private readonly logger: CommonLogger
   private readonly dsOpt: RunQueryOptions
 
   constructor(
     private q: Query,
     opt: DatastoreDBStreamOptions,
-    private logger: CommonLogger,
   ) {
     // 1_000 was optimal in benchmarks
     const { batchSize = 1000 } = opt
@@ -59,10 +59,12 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
       this.dsOpt.readTime = opt.readAt * 1000
     }
 
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+    this.logger = logger
     this.originalLimit = q.limitVal
     this.table = q.kinds[0]!
 
-    logger.warn(`!! using experimentalCursorStream`, {
+    logger.log(`!! using experimentalCursorStream`, {
       table: this.table,
       batchSize,
       highWaterMark,
@@ -70,21 +72,21 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
 
     const { maxWait } = this.opt
     if (maxWait) {
-      logger.warn(`!! ${this.table} maxWait ${maxWait}`)
+      logger.log(`!! ${this.table} maxWait ${maxWait}`)
 
       this.maxWaitInterval = setInterval(
         () => {
           const millisSinceLastRead = Date.now() - this.lastReadTimestamp
 
           if (millisSinceLastRead < maxWait * 1000) {
-            logger.warn(
+            logger.log(
               `!! ${this.table} millisSinceLastRead(${millisSinceLastRead}) < maxWait*1000`,
             )
             return
           }
 
           const { queryIsRunning, rowsRetrieved } = this
-          logger.warn(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
+          logger.log(`maxWait of ${maxWait} seconds reached, force-triggering _read`, {
             running: queryIsRunning,
             rowsRetrieved,
           })
@@ -99,7 +101,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
   }
 
   override _read(): void {
-    this.lastReadTimestamp = Date.now() as UnixTimestampMillis
+    this.lastReadTimestamp = localTime.nowUnixMillis()
 
     // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
     this.countReads++
@@ -117,13 +119,13 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
     }
 
     if (this.queryIsRunning) {
-      this.logger.log(`_read #${this.countReads}, queryIsRunning: true, doing nothing`)
+      this.logger.debug(`_read #${this.countReads}, queryIsRunning: true, doing nothing`)
       return
     }
 
     void this.runNextQuery().catch(err => {
-      console.log('error in runNextQuery', err)
-      this.emit('error', err)
+      this.logger.error('error in runNextQuery', err)
+      this.destroy(err)
     })
   }
 
@@ -192,12 +194,12 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
 
     if (shouldContinue) {
       // Keep the stream flowing
-      logger.log(`${table} continuing the stream`)
+      logger.debug(`${table} continuing the stream`)
       void this.runNextQuery()
     } else {
       // Not starting the next query
       if (this.paused) {
-        logger.log(`${table} stream is already paused`)
+        logger.debug(`${table} stream is already paused`)
       } else {
         logger.log(`${table} pausing the stream`)
         this.paused = true
@@ -215,6 +217,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         },
         {
           name: `DatastoreStreamReadable.query(${table})`,
+          predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
           maxAttempts: 5,
           delay: 5000,
           delayMultiplier: 2,
@@ -223,7 +226,7 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         },
       )
     } catch (err) {
-      console.log(
+      logger.error(
         `DatastoreStreamReadable error!\n`,
         {
           table,
@@ -231,9 +234,22 @@ export class DatastoreStreamReadable<T = any> extends Readable implements Readab
         },
         err,
       )
-      this.emit('error', err)
       clearInterval(this.maxWaitInterval)
+      this.destroy(err as Error)
       return
     }
   }
 }
+
+// Examples of errors:
+// UNKNOWN: Stream removed
+const RETRY_ON = [
+  'GOAWAY',
+  'UNAVAILABLE',
+  'UNKNOWN',
+  'DEADLINE_EXCEEDED',
+  'ABORTED',
+  'much contention',
+  'try again',
+  'timeout',
+].map(s => s.toLowerCase())