@naturalcycles/datastore-lib 3.30.1 → 3.31.0

package/dist/datastore.db.d.ts

@@ -1,5 +1,6 @@
+import { Transaction } from '@google-cloud/datastore';
 import type { Datastore, Key, Query } from '@google-cloud/datastore';
-import { BaseCommonDB, CommonDB, DBQuery, DBTransaction, RunQueryResult } from '@naturalcycles/db-lib';
+import { BaseCommonDB, CommonDB, CommonDBSupport, DBQuery, DBTransaction, RunQueryResult } from '@naturalcycles/db-lib';
 import { ObjectWithId, JsonSchemaObject, JsonSchemaRootObject, CommonLogger } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
 import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats } from './datastore.model';
@@ -9,6 +10,7 @@ import { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDB
  * https://cloud.google.com/datastore/docs/datastore-api-tutorial
  */
 export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
+    support: CommonDBSupport;
     constructor(cfg?: DatastoreDBCfg);
     cfg: DatastoreDBCfg & {
         logger: CommonLogger;
@@ -20,8 +22,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     protected KEY: symbol;
     ds(): Datastore;
     ping(): Promise<void>;
-    getByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][], _opt?: DatastoreDBOptions): Promise<ROW[]>;
-    getQueryKind(q: Query): string;
+    getByIds<ROW extends ObjectWithId>(table: string, ids: string[], _opt?: DatastoreDBOptions): Promise<ROW[]>;
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<number>;
     runDatastoreQuery<ROW extends ObjectWithId>(q: Query): Promise<RunQueryResult<ROW>>;
@@ -36,11 +37,8 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
      * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
      * regardless if they were actually deleted or not.
      */
-    deleteByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][], opt?: DatastoreDBOptions): Promise<number>;
-    /**
-     * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
-     */
-    commitTransaction(_tx: DBTransaction, opt?: DatastoreDBSaveOptions): Promise<void>;
+    deleteByIds(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<number>;
+    createTransaction(): Promise<DatastoreDBTransaction>;
     getAllStats(): Promise<DatastoreStats[]>;
     /**
      * Returns undefined e.g when Table is non-existing
@@ -60,3 +58,14 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;
     private getPRetryOptions;
 }
+/**
+ * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
+ */
+export declare class DatastoreDBTransaction implements DBTransaction {
+    db: DatastoreDB;
+    tx: Transaction;
+    private constructor();
+    static create(db: DatastoreDB): Promise<DatastoreDBTransaction>;
+    commit(): Promise<void>;
+    rollback(): Promise<void>;
+}

package/dist/datastore.db.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DatastoreDB = void 0;
+exports.DatastoreDBTransaction = exports.DatastoreDB = void 0;
 const node_stream_1 = require("node:stream");
 const datastore_1 = require("@google-cloud/datastore");
 const db_lib_1 = require("@naturalcycles/db-lib");
@@ -35,6 +35,10 @@ const methodMap = {
 class DatastoreDB extends db_lib_1.BaseCommonDB {
     constructor(cfg = {}) {
         super();
+        this.support = {
+            ...db_lib_1.commonDBFullSupport,
+            updateByQuery: false,
+        };
         this.cfg = {
             logger: console,
             ...cfg,
@@ -109,11 +113,10 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             // same ids are not expected here
             .sort((a, b) => (a.id > b.id ? 1 : -1)));
     }
-    getQueryKind(q) {
-        if (!q?.kinds?.length)
-            return ''; // should never be the case, but
-        return q.kinds[0];
-    }
+    // getQueryKind(q: Query): string {
+    //   if (!q?.kinds?.length) return '' // should never be the case, but
+    //   return q.kinds[0]!
+    // }
     async runQuery(dbQuery, _opt) {
         const idFilter = dbQuery._filters.find(f => f.name === 'id');
         if (idFilter) {
@@ -172,7 +175,7 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         const entities = rows.map(obj => this.toDatastoreEntity(table, obj, opt.excludeFromIndexes));
         const method = methodMap[opt.saveMethod || 'upsert'] || 'save';
         const save = (0, js_lib_1.pRetryFn)(async (batch) => {
-            await (opt.tx || this.ds())[method](batch);
+            await (opt.tx?.tx || this.ds())[method](batch);
         }, this.getPRetryOptions(`DatastoreLib.saveBatch(${table})`));
         try {
             const chunks = (0, js_lib_1._chunk)(entities, MAX_ITEMS);
@@ -206,38 +209,13 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
      */
     async deleteByIds(table, ids, opt = {}) {
         const keys = ids.map(id => this.key(table, id));
+        await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(keys, MAX_ITEMS), 
         // eslint-disable-next-line @typescript-eslint/return-await
-        await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(keys, MAX_ITEMS), async (batch) => await (opt.tx || this.ds()).delete(batch));
+        async (batch) => await (opt.tx?.tx || this.ds()).delete(batch));
         return ids.length;
     }
-    /**
-     * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
-     */
-    async commitTransaction(_tx, opt) {
-        // Using Retry, because Datastore can throw errors like "too much contention" here
-        await (0, js_lib_1.pRetry)(async () => {
-            const tx = this.ds().transaction();
-            try {
-                await tx.run();
-                const ops = (0, db_lib_1.mergeDBOperations)(_tx.ops);
-                for await (const op of ops) {
-                    if (op.type === 'saveBatch') {
-                        await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt, tx });
-                    }
-                    else if (op.type === 'deleteByIds') {
-                        await this.deleteByIds(op.table, op.ids, { ...op.opt, ...opt, tx });
-                    }
-                    else {
-                        throw new Error(`DBOperation not supported: ${op.type}`);
-                    }
-                }
-                await tx.commit();
-            }
-            catch (err) {
-                await tx.rollback();
-                throw err; // rethrow
-            }
-        }, this.getPRetryOptions(`DatastoreLib.commitTransaction`));
+    async createTransaction() {
+        return await DatastoreDBTransaction.create(this);
     }
     async getAllStats() {
         const q = this.ds().createQuery('__Stat_Kind__');
@@ -393,3 +371,24 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
     }
 }
 exports.DatastoreDB = DatastoreDB;
+/**
+ * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
+ */
+class DatastoreDBTransaction {
+    constructor(db, tx) {
+        this.db = db;
+        this.tx = tx;
+    }
+    static async create(db) {
+        const tx = db.ds().transaction();
+        await tx.run();
+        return new DatastoreDBTransaction(db, tx);
+    }
+    async commit() {
+        await this.tx.commit();
+    }
+    async rollback() {
+        await this.tx.rollback();
+    }
+}
+exports.DatastoreDBTransaction = DatastoreDBTransaction;

package/dist/datastore.model.d.ts

@@ -1,4 +1,4 @@
-import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore';
+import type { DatastoreOptions, Key } from '@google-cloud/datastore';
 import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib';
 import { CommonLogger, NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib';
 export interface DatastorePayload<T = any> {
@@ -100,10 +100,8 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
     maxWait?: NumberOfSeconds;
 }
 export interface DatastoreDBOptions extends CommonDBOptions {
-    tx?: Transaction;
 }
 export interface DatastoreDBSaveOptions<ROW extends Partial<ObjectWithId> = any> extends CommonDBSaveOptions<ROW> {
-    tx?: Transaction;
 }
 export interface DatastoreStats {
     composite_index_count: number;
@@ -116,12 +114,12 @@ export interface DatastoreStats {
 export declare enum DatastoreType {
     Blob = "Blob",
     Text = "Text",
-    String = "String",
+    String = "String",// eslint-disable-line id-blacklist
     EmbeddedEntity = "EmbeddedEntity",
     Float = "Float",
     Integer = "Integer",
     DATE_TIME = "Date/Time",
-    Boolean = "Boolean",
+    Boolean = "Boolean",// eslint-disable-line id-blacklist
     NULL = "NULL"
 }
 export interface DatastorePropertyStats {

package/dist/index.d.ts

@@ -1,5 +1,3 @@
-import { DatastoreDB } from './datastore.db';
-import { DatastoreCredentials, DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats, DatastoreType } from './datastore.model';
-import { DatastoreKeyValueDB, DatastoreKeyValueDBCfg } from './datastoreKeyValueDB';
-export type { DatastorePayload, DatastoreDBCfg, DatastoreKeyValueDBCfg, DatastoreStats, DatastoreCredentials, DatastorePropertyStats, DatastoreDBStreamOptions, DatastoreDBOptions, DatastoreDBSaveOptions, };
-export { DatastoreDB, DatastoreType, DatastoreKeyValueDB };
+export * from './datastore.db';
+export * from './datastore.model';
+export * from './datastoreKeyValueDB';

package/dist/index.js

@@ -1,9 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DatastoreKeyValueDB = exports.DatastoreType = exports.DatastoreDB = void 0;
-const datastore_db_1 = require("./datastore.db");
-Object.defineProperty(exports, "DatastoreDB", { enumerable: true, get: function () { return datastore_db_1.DatastoreDB; } });
-const datastore_model_1 = require("./datastore.model");
-Object.defineProperty(exports, "DatastoreType", { enumerable: true, get: function () { return datastore_model_1.DatastoreType; } });
-const datastoreKeyValueDB_1 = require("./datastoreKeyValueDB");
-Object.defineProperty(exports, "DatastoreKeyValueDB", { enumerable: true, get: function () { return datastoreKeyValueDB_1.DatastoreKeyValueDB; } });
+const tslib_1 = require("tslib");
+tslib_1.__exportStar(require("./datastore.db"), exports);
+tslib_1.__exportStar(require("./datastore.model"), exports);
+tslib_1.__exportStar(require("./datastoreKeyValueDB"), exports);

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.30.1",
+  "version": "3.31.0",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"
   },
   "dependencies": {
     "@google-cloud/datastore": "^8.0.0",
-    "@naturalcycles/db-lib": "^8.46.1",
+    "@naturalcycles/db-lib": "^9.0.0",
     "@naturalcycles/js-lib": "^14.116.0",
     "@naturalcycles/nodejs-lib": "^13.1.0"
   },

package/src/datastore.db.ts

@@ -1,13 +1,14 @@
 import { Transform } from 'node:stream'
-import { PropertyFilter } from '@google-cloud/datastore'
+import { PropertyFilter, Transaction } from '@google-cloud/datastore'
 import type { Datastore, Key, Query } from '@google-cloud/datastore'
 import {
   BaseCommonDB,
   CommonDB,
+  commonDBFullSupport,
   CommonDBSaveMethod,
+  CommonDBSupport,
   DBQuery,
   DBTransaction,
-  mergeDBOperations,
   RunQueryResult,
 } from '@naturalcycles/db-lib'
 import {
@@ -72,6 +73,11 @@ const methodMap: Record<CommonDBSaveMethod, string> = {
  * https://cloud.google.com/datastore/docs/datastore-api-tutorial
  */
 export class DatastoreDB extends BaseCommonDB implements CommonDB {
+  override support: CommonDBSupport = {
+    ...commonDBFullSupport,
+    updateByQuery: false,
+  }
+
   constructor(cfg: DatastoreDBCfg = {}) {
     super()
     this.cfg = {
@@ -125,7 +131,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
   override async getByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: ROW['id'][],
+    ids: string[],
     _opt?: DatastoreDBOptions,
   ): Promise<ROW[]> {
     if (!ids.length) return []
@@ -178,10 +184,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     )
   }
 
-  getQueryKind(q: Query): string {
-    if (!q?.kinds?.length) return '' // should never be the case, but
-    return q.kinds[0]!
-  }
+  // getQueryKind(q: Query): string {
+  //   if (!q?.kinds?.length) return '' // should never be the case, but
+  //   return q.kinds[0]!
+  // }
 
   override async runQuery<ROW extends ObjectWithId>(
     dbQuery: DBQuery<ROW>,
@@ -284,7 +290,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
     const save = pRetryFn(
       async (batch: DatastorePayload<ROW>[]) => {
-        await (opt.tx || this.ds())[method](batch)
+        await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds())[method](batch)
       },
       this.getPRetryOptions(`DatastoreLib.saveBatch(${table})`),
     )
@@ -331,49 +337,23 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
    * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
    * regardless if they were actually deleted or not.
    */
-  async deleteByIds<ROW extends ObjectWithId>(
+  override async deleteByIds(
     table: string,
-    ids: ROW['id'][],
+    ids: string[],
     opt: DatastoreDBOptions = {},
   ): Promise<number> {
     const keys = ids.map(id => this.key(table, id))
-    // eslint-disable-next-line @typescript-eslint/return-await
-    await pMap(_chunk(keys, MAX_ITEMS), async batch => await (opt.tx || this.ds()).delete(batch))
+
+    await pMap(
+      _chunk(keys, MAX_ITEMS),
+      // eslint-disable-next-line @typescript-eslint/return-await
+      async batch => await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).delete(batch),
+    )
     return ids.length
   }
 
-  /**
-   * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
-   */
-  override async commitTransaction(
-    _tx: DBTransaction,
-    opt?: DatastoreDBSaveOptions,
-  ): Promise<void> {
-    // Using Retry, because Datastore can throw errors like "too much contention" here
-    await pRetry(async () => {
-      const tx = this.ds().transaction()
-
-      try {
-        await tx.run()
-
-        const ops = mergeDBOperations(_tx.ops)
-
-        for await (const op of ops) {
-          if (op.type === 'saveBatch') {
-            await this.saveBatch(op.table, op.rows, { ...op.opt, ...opt, tx })
-          } else if (op.type === 'deleteByIds') {
-            await this.deleteByIds(op.table, op.ids, { ...op.opt, ...opt, tx })
-          } else {
-            throw new Error(`DBOperation not supported: ${(op as any).type}`)
-          }
-        }
-
-        await tx.commit()
-      } catch (err) {
-        await tx.rollback()
-        throw err // rethrow
-      }
-    }, this.getPRetryOptions(`DatastoreLib.commitTransaction`))
+  override async createTransaction(): Promise<DatastoreDBTransaction> {
+    return await DatastoreDBTransaction.create(this)
   }
 
   async getAllStats(): Promise<DatastoreStats[]> {
@@ -547,3 +527,26 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 }
+
+/**
+ * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
+ */
+export class DatastoreDBTransaction implements DBTransaction {
+  private constructor(
+    public db: DatastoreDB,
+    public tx: Transaction,
+  ) {}
+
+  static async create(db: DatastoreDB): Promise<DatastoreDBTransaction> {
+    const tx = db.ds().transaction()
+    await tx.run()
+    return new DatastoreDBTransaction(db, tx)
+  }
+
+  async commit(): Promise<void> {
+    await this.tx.commit()
+  }
+  async rollback(): Promise<void> {
+    await this.tx.rollback()
+  }
+}

package/src/datastore.model.ts

@@ -1,4 +1,4 @@
-import type { DatastoreOptions, Key, Transaction } from '@google-cloud/datastore'
+import type { DatastoreOptions, Key } from '@google-cloud/datastore'
 import { CommonDBOptions, CommonDBSaveOptions } from '@naturalcycles/db-lib'
 import { CommonLogger, NumberOfSeconds, ObjectWithId } from '@naturalcycles/js-lib'
 
@@ -114,13 +114,10 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
   maxWait?: NumberOfSeconds
 }
 
-export interface DatastoreDBOptions extends CommonDBOptions {
-  tx?: Transaction
-}
+export interface DatastoreDBOptions extends CommonDBOptions {}
+
 export interface DatastoreDBSaveOptions<ROW extends Partial<ObjectWithId> = any>
-  extends CommonDBSaveOptions<ROW> {
-  tx?: Transaction
-}
+  extends CommonDBSaveOptions<ROW> {}
 
 export interface DatastoreStats {
   composite_index_count: number

package/src/datastoreKeyValueDB.ts

@@ -53,7 +53,7 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       .streamQuery<KVObject>(q)
       .on('error', err => stream.emit('error', err))
       .pipe(
-        transformMapSimple<ObjectWithId<string>, string>(objectWithId => objectWithId.id, {
+        transformMapSimple<ObjectWithId, string>(objectWithId => objectWithId.id, {
           errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
         }),
       )

package/src/index.ts

@@ -1,27 +1,3 @@
-import { DatastoreDB } from './datastore.db'
-import {
-  DatastoreCredentials,
-  DatastoreDBCfg,
-  DatastoreDBOptions,
-  DatastoreDBSaveOptions,
-  DatastoreDBStreamOptions,
-  DatastorePayload,
-  DatastorePropertyStats,
-  DatastoreStats,
-  DatastoreType,
-} from './datastore.model'
-import { DatastoreKeyValueDB, DatastoreKeyValueDBCfg } from './datastoreKeyValueDB'
-
-export type {
-  DatastorePayload,
-  DatastoreDBCfg,
-  DatastoreKeyValueDBCfg,
-  DatastoreStats,
-  DatastoreCredentials,
-  DatastorePropertyStats,
-  DatastoreDBStreamOptions,
-  DatastoreDBOptions,
-  DatastoreDBSaveOptions,
-}
-
-export { DatastoreDB, DatastoreType, DatastoreKeyValueDB }
+export * from './datastore.db'
+export * from './datastore.model'
+export * from './datastoreKeyValueDB'