@naturalcycles/datastore-lib 3.32.2 → 3.33.0

package/dist/DatastoreStreamReadable.js

@@ -80,7 +80,7 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
                 delay: 5000,
                 delayMultiplier: 2,
                 logger: this.logger,
-                timeout: 120000, // 2 minutes
+                timeout: 120_000, // 2 minutes
             });
         }
         catch (err) {

package/dist/datastore.db.d.ts

@@ -1,7 +1,7 @@
 import { Transaction } from '@google-cloud/datastore';
 import type { Datastore, Key, Query } from '@google-cloud/datastore';
 import { BaseCommonDB, CommonDB, CommonDBOptions, CommonDBSaveOptions, CommonDBSupport, DBQuery, DBTransaction, DBTransactionFn, RunQueryResult, CommonDBTransactionOptions } from '@naturalcycles/db-lib';
-import { JsonSchemaObject, JsonSchemaRootObject, CommonLogger, PartialObjectWithId, Saved, ObjectWithId } from '@naturalcycles/js-lib';
+import { JsonSchemaObject, JsonSchemaRootObject, CommonLogger, ObjectWithId } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats } from './datastore.model';
 /**
@@ -22,17 +22,17 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     protected KEY: symbol;
     ds(): Datastore;
     ping(): Promise<void>;
-    getByIds<ROW extends PartialObjectWithId>(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<Saved<ROW>[]>;
-    runQuery<ROW extends PartialObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<RunQueryResult<Saved<ROW>>>;
-    runQueryCount<ROW extends PartialObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<number>;
-    runDatastoreQuery<ROW extends PartialObjectWithId>(q: Query): Promise<RunQueryResult<Saved<ROW>>>;
+    getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<ROW[]>;
+    runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<RunQueryResult<ROW>>;
+    runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<number>;
+    runDatastoreQuery<ROW extends ObjectWithId>(q: Query): Promise<RunQueryResult<ROW>>;
     private runQueryStream;
-    streamQuery<ROW extends PartialObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBStreamOptions): ReadableTyped<ROW>;
     /**
      * Returns saved entities with generated id/updated/created (non-mutating!)
      */
-    saveBatch<ROW extends PartialObjectWithId>(table: string, rows: ROW[], opt?: DatastoreDBSaveOptions<ROW>): Promise<void>;
-    deleteByQuery<ROW extends PartialObjectWithId>(q: DBQuery<ROW>, opt?: DatastoreDBOptions): Promise<number>;
+    saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: DatastoreDBSaveOptions<ROW>): Promise<void>;
+    deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: DatastoreDBOptions): Promise<number>;
     /**
      * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
      * regardless if they were actually deleted or not.
@@ -53,9 +53,9 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     key(kind: string, id: string | number): Key;
     getDsKey(o: any): Key | undefined;
     getKey(key: Key): string | undefined;
-    createTable<ROW extends PartialObjectWithId>(_table: string, _schema: JsonSchemaObject<ROW>): Promise<void>;
+    createTable<ROW extends ObjectWithId>(_table: string, _schema: JsonSchemaObject<ROW>): Promise<void>;
     getTables(): Promise<string[]>;
-    getTableSchema<ROW extends PartialObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;
+    getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;
     private getPRetryOptions;
 }
 /**
@@ -66,7 +66,7 @@ export declare class DatastoreDBTransaction implements DBTransaction {
     tx: Transaction;
     constructor(db: DatastoreDB, tx: Transaction);
     rollback(): Promise<void>;
-    getByIds<ROW extends PartialObjectWithId>(table: string, ids: string[], opt?: CommonDBOptions | undefined): Promise<ROW[]>;
-    saveBatch<ROW extends PartialObjectWithId>(table: string, rows: ROW[], opt?: CommonDBSaveOptions<ROW> | undefined): Promise<void>;
+    getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: CommonDBOptions | undefined): Promise<ROW[]>;
+    saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: CommonDBSaveOptions<ROW> | undefined): Promise<void>;
     deleteByIds(table: string, ids: string[], opt?: CommonDBOptions | undefined): Promise<number>;
 }

package/dist/datastore.db.js

@@ -1,7 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatastoreDBTransaction = exports.DatastoreDB = void 0;
-const node_stream_1 = require("node:stream");
 const datastore_1 = require("@google-cloud/datastore");
 const db_lib_1 = require("@naturalcycles/db-lib");
 const js_lib_1 = require("@naturalcycles/js-lib");
@@ -135,8 +134,9 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
     }
     async runQueryCount(dbQuery, _opt) {
         const q = (0, query_util_1.dbQueryToDatastoreQuery)(dbQuery.select([]), this.ds().createQuery(dbQuery.table));
-        const [entities] = await this.ds().runQuery(q);
-        return entities.length;
+        const aq = this.ds().createAggregationQuery(q).count('count');
+        const [entities] = await this.ds().runAggregationQuery(aq);
+        return entities[0]?.count;
     }
     async runDatastoreQuery(q) {
         const [entities, queryResult] = await this.ds().runQuery(q);
@@ -154,13 +154,8 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         const stream = (opt.experimentalCursorStream
             ? new DatastoreStreamReadable_1.DatastoreStreamReadable(q, opt, (0, js_lib_1.commonLoggerMinLevel)(this.cfg.logger, opt.debug ? 'log' : 'warn'))
             : this.ds().runQueryStream(q))
-            .on('error', err => stream.emit('error', err))
-            .pipe(new node_stream_1.Transform({
-            objectMode: true,
-            transform: (chunk, _enc, cb) => {
-                cb(null, this.mapId(chunk));
-            },
-        }));
+            // .on('error', err => stream.emit('error', err))
+            .map(chunk => this.mapId(chunk));
         return stream;
     }
     streamQuery(dbQuery, opt) {
@@ -369,7 +364,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         return {
             predicate: err => RETRY_ON.some(s => err?.message?.toLowerCase()?.includes(s)),
             name,
-            timeout: 20000,
+            timeout: 20_000,
             maxAttempts: 5,
             delay: 5000,
             delayMultiplier: 1.5,

package/dist/datastore.model.d.ts

@@ -101,7 +101,7 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
 }
 export interface DatastoreDBOptions extends CommonDBOptions {
 }
-export interface DatastoreDBSaveOptions<ROW extends Partial<ObjectWithId> = any> extends CommonDBSaveOptions<ROW> {
+export interface DatastoreDBSaveOptions<ROW extends ObjectWithId> extends CommonDBSaveOptions<ROW> {
 }
 export interface DatastoreStats {
     composite_index_count: number;

package/dist/datastoreKeyValueDB.d.ts

@@ -17,5 +17,5 @@ export declare class DatastoreKeyValueDB implements CommonKeyValueDB {
     streamIds(table: string, limit?: number): ReadableTyped<string>;
     streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
     streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
-    count(_table: string): Promise<number>;
+    count(table: string): Promise<number>;
 }

package/dist/datastoreKeyValueDB.js

@@ -2,8 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DatastoreKeyValueDB = void 0;
 const db_lib_1 = require("@naturalcycles/db-lib");
-const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
 const datastore_db_1 = require("./datastore.db");
 const excludeFromIndexes = ['v'];
 class DatastoreKeyValueDB {
@@ -30,38 +28,29 @@ class DatastoreKeyValueDB {
         const q = db_lib_1.DBQuery.create(table)
             .select(['id'])
             .limit(limit || 0);
-        const stream = this.db
+        return (this.db
             .streamQuery(q)
-            .on('error', err => stream.emit('error', err))
-            .pipe((0, nodejs_lib_1.transformMapSimple)(objectWithId => objectWithId.id, {
-            errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        }));
-        return stream;
+            // .on('error', err => stream.emit('error', err))
+            .map(r => r.id));
     }
     streamValues(table, limit) {
         // `select v` doesn't work for some reason
         const q = db_lib_1.DBQuery.create(table).limit(limit || 0);
-        const stream = this.db
+        return (this.db
             .streamQuery(q)
-            .on('error', err => stream.emit('error', err))
-            .pipe((0, nodejs_lib_1.transformMapSimple)(obj => obj.v, {
-            errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        }));
-        return stream;
+            // .on('error', err => stream.emit('error', err))
+            .map(r => r.v));
     }
     streamEntries(table, limit) {
         const q = db_lib_1.DBQuery.create(table).limit(limit || 0);
-        const stream = this.db
+        return (this.db
             .streamQuery(q)
-            .on('error', err => stream.emit('error', err))
-            .pipe((0, nodejs_lib_1.transformMapSimple)(obj => [obj.id, obj.v], {
-            errorMode: js_lib_1.ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        }));
-        return stream;
+            // .on('error', err => stream.emit('error', err))
+            .map(r => [r.id, r.v]));
     }
-    async count(_table) {
-        this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`);
-        return 0;
+    async count(table) {
+        const q = db_lib_1.DBQuery.create(table);
+        return await this.db.runQueryCount(q);
     }
 }
 exports.DatastoreKeyValueDB = DatastoreKeyValueDB;

package/dist/query.util.d.ts

@@ -1,4 +1,4 @@
 import { Query } from '@google-cloud/datastore';
 import { DBQuery } from '@naturalcycles/db-lib';
-import { PartialObjectWithId } from '@naturalcycles/js-lib';
-export declare function dbQueryToDatastoreQuery<ROW extends PartialObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query): Query;
+import { ObjectWithId } from '@naturalcycles/js-lib';
+export declare function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(dbQuery: Readonly<DBQuery<ROW>>, emptyQuery: Query): Query;

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.32.2",
+  "version": "3.33.0",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
-    "prepare": "husky install"
+    "prepare": "husky"
   },
   "dependencies": {
     "@google-cloud/datastore": "^8.0.0",

package/src/datastore.db.ts

@@ -1,4 +1,3 @@
-import { Transform } from 'node:stream'
 import { PropertyFilter, Transaction } from '@google-cloud/datastore'
 import type { Datastore, Key, Query } from '@google-cloud/datastore'
 import {
@@ -33,8 +32,6 @@ import {
   pRetryFn,
   pRetry,
   PRetryOptions,
-  PartialObjectWithId,
-  Saved,
   ObjectWithId,
 } from '@naturalcycles/js-lib'
 import { ReadableTyped, boldWhite } from '@naturalcycles/nodejs-lib'
@@ -135,11 +132,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     await this.getAllStats()
   }
 
-  override async getByIds<ROW extends PartialObjectWithId>(
+  override async getByIds<ROW extends ObjectWithId>(
     table: string,
     ids: string[],
     opt: DatastoreDBOptions = {},
-  ): Promise<Saved<ROW>[]> {
+  ): Promise<ROW[]> {
     if (!ids.length) return []
     const keys = ids.map(id => this.key(table, id))
     let rows: any[]
@@ -189,7 +186,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
     return (
       rows
-        .map(r => this.mapId<Saved<ROW>>(r))
+        .map(r => this.mapId<ROW>(r))
         // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
         // same ids are not expected here
         .sort((a, b) => (a.id > b.id ? 1 : -1))
@@ -201,10 +198,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   //   return q.kinds[0]!
   // }
 
-  override async runQuery<ROW extends PartialObjectWithId>(
+  override async runQuery<ROW extends ObjectWithId>(
     dbQuery: DBQuery<ROW>,
     _opt?: DatastoreDBOptions,
-  ): Promise<RunQueryResult<Saved<ROW>>> {
+  ): Promise<RunQueryResult<ROW>> {
     const idFilter = dbQuery._filters.find(f => f.name === 'id')
     if (idFilter) {
       const ids: string[] = idFilter.op === '==' ? [idFilter.val] : idFilter.val
@@ -225,21 +222,20 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     return qr
   }
 
-  override async runQueryCount<ROW extends PartialObjectWithId>(
+  override async runQueryCount<ROW extends ObjectWithId>(
     dbQuery: DBQuery<ROW>,
     _opt?: DatastoreDBOptions,
   ): Promise<number> {
     const q = dbQueryToDatastoreQuery(dbQuery.select([]), this.ds().createQuery(dbQuery.table))
-    const [entities] = await this.ds().runQuery(q)
-    return entities.length
+    const aq = this.ds().createAggregationQuery(q).count('count')
+    const [entities] = await this.ds().runAggregationQuery(aq)
+    return entities[0]?.count
   }
 
-  async runDatastoreQuery<ROW extends PartialObjectWithId>(
-    q: Query,
-  ): Promise<RunQueryResult<Saved<ROW>>> {
+  async runDatastoreQuery<ROW extends ObjectWithId>(q: Query): Promise<RunQueryResult<ROW>> {
     const [entities, queryResult] = await this.ds().runQuery(q)
 
-    const rows = entities.map(e => this.mapId<Saved<ROW>>(e))
+    const rows = entities.map(e => this.mapId<ROW>(e))
 
     return {
       ...queryResult,
@@ -247,7 +243,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  private runQueryStream<ROW extends PartialObjectWithId>(
+  private runQueryStream<ROW extends ObjectWithId>(
     q: Query,
     _opt?: DatastoreDBStreamOptions,
   ): ReadableTyped<ROW> {
@@ -258,27 +254,20 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
     const stream: ReadableTyped<ROW> = (
       opt.experimentalCursorStream
-        ? new DatastoreStreamReadable(
+        ? new DatastoreStreamReadable<ROW>(
             q,
             opt,
             commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
           )
         : this.ds().runQueryStream(q)
     )
-      .on('error', err => stream.emit('error', err))
-      .pipe(
-        new Transform({
-          objectMode: true,
-          transform: (chunk, _enc, cb) => {
-            cb(null, this.mapId(chunk))
-          },
-        }),
-      )
+      // .on('error', err => stream.emit('error', err))
+      .map(chunk => this.mapId(chunk))
 
     return stream
   }
 
-  override streamQuery<ROW extends PartialObjectWithId>(
+  override streamQuery<ROW extends ObjectWithId>(
     dbQuery: DBQuery<ROW>,
     opt?: DatastoreDBStreamOptions,
   ): ReadableTyped<ROW> {
@@ -291,7 +280,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   /**
    * Returns saved entities with generated id/updated/created (non-mutating!)
    */
-  override async saveBatch<ROW extends PartialObjectWithId>(
+  override async saveBatch<ROW extends ObjectWithId>(
     table: string,
     rows: ROW[],
     opt: DatastoreDBSaveOptions<ROW> = {},
@@ -328,7 +317,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  override async deleteByQuery<ROW extends PartialObjectWithId>(
+  override async deleteByQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     opt?: DatastoreDBOptions,
   ): Promise<number> {
@@ -462,7 +451,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     return id?.toString()
   }
 
-  override async createTable<ROW extends PartialObjectWithId>(
+  override async createTable<ROW extends ObjectWithId>(
     _table: string,
     _schema: JsonSchemaObject<ROW>,
   ): Promise<void> {}
@@ -473,7 +462,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     return statsArray.map(stats => stats.kind_name).filter(table => table && !table.startsWith('_'))
   }
 
-  override async getTableSchema<ROW extends PartialObjectWithId>(
+  override async getTableSchema<ROW extends ObjectWithId>(
     table: string,
   ): Promise<JsonSchemaRootObject<ROW>> {
     const stats = await this.getTableProperties(table)
@@ -576,7 +565,7 @@ export class DatastoreDBTransaction implements DBTransaction {
     }
   }
 
-  async getByIds<ROW extends PartialObjectWithId>(
+  async getByIds<ROW extends ObjectWithId>(
     table: string,
     ids: string[],
     opt?: CommonDBOptions | undefined,
@@ -584,7 +573,7 @@ export class DatastoreDBTransaction implements DBTransaction {
     return await this.db.getByIds(table, ids, { ...opt, tx: this })
   }
 
-  async saveBatch<ROW extends PartialObjectWithId>(
+  async saveBatch<ROW extends ObjectWithId>(
     table: string,
     rows: ROW[],
     opt?: CommonDBSaveOptions<ROW> | undefined,

package/src/datastore.model.ts

@@ -116,7 +116,7 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
 
 export interface DatastoreDBOptions extends CommonDBOptions {}
 
-export interface DatastoreDBSaveOptions<ROW extends Partial<ObjectWithId> = any>
+export interface DatastoreDBSaveOptions<ROW extends ObjectWithId>
   extends CommonDBSaveOptions<ROW> {}
 
 export interface DatastoreStats {

package/src/datastoreKeyValueDB.ts

@@ -1,6 +1,5 @@
 import { CommonKeyValueDB, DBQuery, KeyValueDBTuple } from '@naturalcycles/db-lib'
-import { ErrorMode, ObjectWithId } from '@naturalcycles/js-lib'
-import { ReadableTyped, transformMapSimple } from '@naturalcycles/nodejs-lib'
+import { ReadableTyped } from '@naturalcycles/nodejs-lib'
 import { DatastoreDB } from './datastore.db'
 import { DatastoreDBCfg } from './datastore.model'
 
@@ -49,51 +48,39 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       .select(['id'])
       .limit(limit || 0)
 
-    const stream: ReadableTyped<string> = this.db
-      .streamQuery<KVObject>(q)
-      .on('error', err => stream.emit('error', err))
-      .pipe(
-        transformMapSimple<ObjectWithId, string>(objectWithId => objectWithId.id, {
-          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        }),
-      )
-
-    return stream
+    return (
+      this.db
+        .streamQuery(q)
+        // .on('error', err => stream.emit('error', err))
+        .map(r => r.id)
+    )
   }
 
   streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
     // `select v` doesn't work for some reason
     const q = DBQuery.create<KVObject>(table).limit(limit || 0)
 
-    const stream: ReadableTyped<string> = this.db
-      .streamQuery<KVObject>(q)
-      .on('error', err => stream.emit('error', err))
-      .pipe(
-        transformMapSimple<{ v: Buffer }, Buffer>(obj => obj.v, {
-          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        }),
-      )
-
-    return stream
+    return (
+      this.db
+        .streamQuery(q)
+        // .on('error', err => stream.emit('error', err))
+        .map(r => r.v)
+    )
   }
 
   streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
     const q = DBQuery.create<KVObject>(table).limit(limit || 0)
 
-    const stream: ReadableTyped<string> = this.db
-      .streamQuery<KVObject>(q)
-      .on('error', err => stream.emit('error', err))
-      .pipe(
-        transformMapSimple<KVObject, KeyValueDBTuple>(obj => [obj.id, obj.v], {
-          errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
-        }),
-      )
-
-    return stream
+    return (
+      this.db
+        .streamQuery(q)
+        // .on('error', err => stream.emit('error', err))
+        .map(r => [r.id, r.v] as KeyValueDBTuple)
+    )
   }
 
-  async count(_table: string): Promise<number> {
-    this.db.cfg.logger.warn(`DatastoreKeyValueDB.count is not supported`)
-    return 0
+  async count(table: string): Promise<number> {
+    const q = DBQuery.create<KVObject>(table)
+    return await this.db.runQueryCount(q)
   }
 }

package/src/query.util.ts

@@ -1,6 +1,6 @@
 import { PropertyFilter, Query } from '@google-cloud/datastore'
 import { DBQuery, DBQueryFilterOperator } from '@naturalcycles/db-lib'
-import { PartialObjectWithId, StringMap } from '@naturalcycles/js-lib'
+import { ObjectWithId, StringMap } from '@naturalcycles/js-lib'
 
 const FNAME_MAP: StringMap = {
   id: '__key__',
@@ -12,7 +12,7 @@ const OP_MAP: Partial<Record<DBQueryFilterOperator, string>> = {
   'not-in': 'NOT_IN',
 }
 
-export function dbQueryToDatastoreQuery<ROW extends PartialObjectWithId>(
+export function dbQueryToDatastoreQuery<ROW extends ObjectWithId>(
   dbQuery: Readonly<DBQuery<ROW>>,
   emptyQuery: Query,
 ): Query {