@naturalcycles/datastore-lib 4.10.0 → 4.12.0

package/dist/datastore.db.d.ts

@@ -3,7 +3,7 @@ import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOption
 import { BaseCommonDB } from '@naturalcycles/db-lib';
 import type { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib/json-schema';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import type { ObjectWithId } from '@naturalcycles/js-lib/types';
+import { type ObjectWithId, type StringMap } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
 import type { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBReadOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePropertyStats, DatastoreStats } from './datastore.model.js';
 /**
@@ -27,6 +27,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     private getDatastoreLib;
     ping(): Promise<void>;
     getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: DatastoreDBReadOptions): Promise<ROW[]>;
+    multiGetByIds<ROW extends ObjectWithId>(map: StringMap<string[]>, opt?: DatastoreDBReadOptions): Promise<StringMap<ROW[]>>;
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<number>;
     private runDatastoreQuery;
@@ -41,6 +42,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
      * regardless if they were actually deleted or not.
      */
     deleteByIds(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<number>;
+    multiDeleteByIds(map: StringMap<string[]>, opt?: DatastoreDBOptions): Promise<number>;
     createTransaction(opt?: CommonDBTransactionOptions): Promise<DatastoreDBTransaction>;
     runInTransaction(fn: DBTransactionFn, opt?: CommonDBTransactionOptions): Promise<void>;
     getAllStats(): Promise<DatastoreStats[]>;
@@ -51,10 +53,11 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     getStatsCount(table: string): Promise<number | undefined>;
     getTableProperties(table: string): Promise<DatastorePropertyStats[]>;
     private mapId;
+    private parseDatastoreEntity;
     private toDatastoreEntity;
     key(ds: Datastore, kind: string, id: string): Key;
     private getDsKey;
-    getKey(key: Key): string | undefined;
+    private getIdFromKey;
     createTable<ROW extends ObjectWithId>(_table: string, _schema: JsonSchemaObject<ROW>): Promise<void>;
     getTables(): Promise<string[]>;
     getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;

package/dist/datastore.db.js

@@ -9,6 +9,7 @@ import { _omit } from '@naturalcycles/js-lib/object/object.util.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 import { pRetry, pRetryFn } from '@naturalcycles/js-lib/promise/pRetry.js';
 import { pTimeout } from '@naturalcycles/js-lib/promise/pTimeout.js';
+import { _stringMapEntries, _stringMapValues, } from '@naturalcycles/js-lib/types';
 import { boldWhite } from '@naturalcycles/nodejs-lib/colors';
 import { DatastoreType } from './datastore.model.js';
 import { DatastoreStreamReadable } from './DatastoreStreamReadable.js';
@@ -44,6 +45,7 @@ export class DatastoreDB extends BaseCommonDB {
     support = {
         ...commonDBFullSupport,
         patchByQuery: false,
+        patchById: false, // use Firestore for that
         increment: false,
     };
     constructor(cfg = {}) {
@@ -128,7 +130,7 @@ export class DatastoreDB extends BaseCommonDB {
                 catch (err) {
                     if (err instanceof TimeoutError) {
                         _errorDataAppend(err, {
-                            fingerprint: [DATASTORE_TIMEOUT],
+                            fingerprint: DATASTORE_TIMEOUT,
                         });
                     }
                     throw err;
@@ -144,7 +146,29 @@ export class DatastoreDB extends BaseCommonDB {
             .map(r => this.mapId(r))
             // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
             // same ids are not expected here
-            .sort((a, b) => (a.id > b.id ? 1 : -1)));
+            .sort(idComparator));
+    }
+    async multiGetByIds(map, opt = {}) {
+        const result = {};
+        const ds = await this.ds();
+        const dsOpt = this.getRunQueryOptions(opt);
+        const keys = [];
+        for (const [table, ids] of _stringMapEntries(map)) {
+            result[table] = [];
+            keys.push(...ids.map(id => this.key(ds, table, id)));
+        }
+        const r = await ds.get(keys, dsOpt);
+        const rows = r[0];
+        rows.forEach(entity => {
+            const [kind, row] = this.parseDatastoreEntity(entity);
+            result[kind].push(row);
+        });
+        // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
+        // same ids are not expected here
+        for (const tableRows of _stringMapValues(result)) {
+            tableRows.sort(idComparator);
+        }
+        return result;
     }
     // getQueryKind(q: Query): string {
     //   if (!q?.kinds?.length) return '' // should never be the case, but
@@ -232,7 +256,7 @@ export class DatastoreDB extends BaseCommonDB {
         catch (err) {
             if (err instanceof TimeoutError) {
                 _errorDataAppend(err, {
-                    fingerprint: [DATASTORE_TIMEOUT],
+                    fingerprint: DATASTORE_TIMEOUT,
                 });
             }
             // console.log(`datastore.save ${kind}`, { obj, entity })
@@ -240,6 +264,7 @@ export class DatastoreDB extends BaseCommonDB {
             throw err;
         }
     }
+    // not implementing multiSaveBatch, since the API does not support passing excludeFromIndexes for multiple tables
     async deleteByQuery(q, opt = {}) {
         const idFilter = q._filters.find(f => f.name === 'id');
         if (idFilter) {
@@ -271,6 +296,24 @@ export class DatastoreDB extends BaseCommonDB {
         });
         return ids.length;
     }
+    async multiDeleteByIds(map, opt = {}) {
+        const ds = await this.ds();
+        const keys = [];
+        for (const [table, ids] of _stringMapEntries(map)) {
+            keys.push(...ids.map(id => this.key(ds, table, id)));
+        }
+        const retryOptions = this.getPRetryOptions(`DatastoreLib.multiDeleteByIds`);
+        await pMap(_chunk(keys, MAX_ITEMS), 
+        // async batch => await doDelete(batch),
+        async (batchOfKeys) => {
+            await pRetry(async () => {
+                await (opt.tx?.tx || ds).delete(batchOfKeys);
+            }, retryOptions);
+        }, {
+            concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
+        });
+        return keys.length;
+    }
     async createTransaction(opt = {}) {
         const ds = await this.ds();
         const { readOnly } = opt;
@@ -336,11 +379,21 @@ export class DatastoreDB extends BaseCommonDB {
             return o;
         const r = {
             ...o,
-            id: this.getKey(this.getDsKey(o)),
+            id: this.getIdFromKey(this.getDsKey(o)),
         };
         delete r[this.KEY];
         return r;
     }
+    parseDatastoreEntity(entity) {
+        const key = this.getDsKey(entity);
+        const { name, kind } = key;
+        const row = {
+            ...entity,
+            id: name,
+        };
+        delete row[this.KEY];
+        return [kind, row];
+    }
     // if key field exists on entity, it will be used as key (prevent to duplication of numeric keyed entities)
     toDatastoreEntity(ds, kind, o, excludeFromIndexes = []) {
         const key = this.getDsKey(o) || this.key(ds, kind, o.id);
@@ -360,7 +413,7 @@ export class DatastoreDB extends BaseCommonDB {
     getDsKey(o) {
         return o?.[this.KEY];
     }
-    getKey(key) {
+    getIdFromKey(key) {
         const id = key.id || key.name;
         return id?.toString();
     }
@@ -449,7 +502,7 @@ export class DatastoreDB extends BaseCommonDB {
             logger: this.cfg.logger,
             // not appending fingerprint here, otherwise it would just group all kinds of errors, not just Timeout errors
             // errorData: {
-            //   fingerprint: [DATASTORE_TIMEOUT],
+            //   fingerprint: DATASTORE_TIMEOUT,
             // },
         };
     }
@@ -508,3 +561,6 @@ export class DatastoreDBTransaction {
         return await this.db.deleteByIds(table, ids, { ...opt, tx: this });
     }
 }
+function idComparator(a, b) {
+    return a.id > b.id ? 1 : a.id < b.id ? -1 : 0;
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/datastore-lib",
   "type": "module",
-  "version": "4.10.0",
+  "version": "4.12.0",
   "description": "Opinionated library to work with Google Datastore, implements CommonDB",
   "dependencies": {
     "@google-cloud/datastore": "^10",

package/src/datastore.db.ts

@@ -35,7 +35,12 @@ import type { PRetryOptions } from '@naturalcycles/js-lib/promise'
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
 import { pRetry, pRetryFn } from '@naturalcycles/js-lib/promise/pRetry.js'
 import { pTimeout } from '@naturalcycles/js-lib/promise/pTimeout.js'
-import type { ObjectWithId } from '@naturalcycles/js-lib/types'
+import {
+  _stringMapEntries,
+  _stringMapValues,
+  type ObjectWithId,
+  type StringMap,
+} from '@naturalcycles/js-lib/types'
 import { boldWhite } from '@naturalcycles/nodejs-lib/colors'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
 import type {
@@ -87,6 +92,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   override support: CommonDBSupport = {
     ...commonDBFullSupport,
     patchByQuery: false,
+    patchById: false, // use Firestore for that
     increment: false,
   }
 
@@ -199,7 +205,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         } catch (err) {
           if (err instanceof TimeoutError) {
             _errorDataAppend(err, {
-              fingerprint: [DATASTORE_TIMEOUT],
+              fingerprint: DATASTORE_TIMEOUT,
             })
           }
           throw err
@@ -219,10 +225,40 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         .map(r => this.mapId<ROW>(r))
         // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
         // same ids are not expected here
-        .sort((a, b) => (a.id > b.id ? 1 : -1))
+        .sort(idComparator)
     )
   }
 
+  override async multiGetByIds<ROW extends ObjectWithId>(
+    map: StringMap<string[]>,
+    opt: DatastoreDBReadOptions = {},
+  ): Promise<StringMap<ROW[]>> {
+    const result: StringMap<ROW[]> = {}
+    const ds = await this.ds()
+    const dsOpt = this.getRunQueryOptions(opt)
+    const keys: Key[] = []
+    for (const [table, ids] of _stringMapEntries(map)) {
+      result[table] = []
+      keys.push(...ids.map(id => this.key(ds, table, id)))
+    }
+
+    const r = await ds.get(keys, dsOpt)
+    const rows: any[] = r[0]
+
+    rows.forEach(entity => {
+      const [kind, row] = this.parseDatastoreEntity<ROW>(entity)
+      result[kind]!.push(row)
+    })
+
+    // Seems like datastore .get() method doesn't return items properly sorted by input ids, so we gonna sort them here
+    // same ids are not expected here
+    for (const tableRows of _stringMapValues(result)) {
+      tableRows.sort(idComparator)
+    }
+
+    return result
+  }
+
   // getQueryKind(q: Query): string {
   //   if (!q?.kinds?.length) return '' // should never be the case, but
   //   return q.kinds[0]!
@@ -364,7 +400,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     } catch (err) {
       if (err instanceof TimeoutError) {
         _errorDataAppend(err, {
-          fingerprint: [DATASTORE_TIMEOUT],
+          fingerprint: DATASTORE_TIMEOUT,
         })
       }
 
@@ -378,6 +414,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 
+  // not implementing multiSaveBatch, since the API does not support passing excludeFromIndexes for multiple tables
+
   override async deleteByQuery<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     opt: DatastoreDBReadOptions = {},
@@ -432,6 +470,34 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     return ids.length
   }
 
+  override async multiDeleteByIds(
+    map: StringMap<string[]>,
+    opt: DatastoreDBOptions = {},
+  ): Promise<number> {
+    const ds = await this.ds()
+    const keys: Key[] = []
+    for (const [table, ids] of _stringMapEntries(map)) {
+      keys.push(...ids.map(id => this.key(ds, table, id)))
+    }
+
+    const retryOptions = this.getPRetryOptions(`DatastoreLib.multiDeleteByIds`)
+
+    await pMap(
+      _chunk(keys, MAX_ITEMS),
+      // async batch => await doDelete(batch),
+      async batchOfKeys => {
+        await pRetry(async () => {
+          await ((opt.tx as DatastoreDBTransaction)?.tx || ds).delete(batchOfKeys)
+        }, retryOptions)
+      },
+      {
+        concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
+      },
+    )
+
+    return keys.length
+  }
+
   override async createTransaction(
     opt: CommonDBTransactionOptions = {},
   ): Promise<DatastoreDBTransaction> {
@@ -508,12 +574,23 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     if (!o) return o
     const r = {
       ...o,
-      id: this.getKey(this.getDsKey(o)!),
+      id: this.getIdFromKey(this.getDsKey(o)!),
     }
     delete r[this.KEY]
     return r
   }
 
+  private parseDatastoreEntity<T extends ObjectWithId>(entity: any): [kind: string, row: T] {
+    const key = this.getDsKey(entity)!
+    const { name, kind } = key
+    const row: any = {
+      ...entity,
+      id: name,
+    }
+    delete row[this.KEY]
+    return [kind, row]
+  }
+
   // if key field exists on entity, it will be used as key (prevent to duplication of numeric keyed entities)
   private toDatastoreEntity<T extends ObjectWithId>(
     ds: Datastore,
@@ -542,7 +619,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     return o?.[this.KEY]
   }
 
-  getKey(key: Key): string | undefined {
+  private getIdFromKey(key: Key): string | undefined {
     const id = key.id || key.name
     return id?.toString()
   }
@@ -638,7 +715,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       logger: this.cfg.logger,
       // not appending fingerprint here, otherwise it would just group all kinds of errors, not just Timeout errors
       // errorData: {
-      //   fingerprint: [DATASTORE_TIMEOUT],
+      //   fingerprint: DATASTORE_TIMEOUT,
       // },
     }
   }
@@ -709,3 +786,7 @@ export class DatastoreDBTransaction implements DBTransaction {
     return await this.db.deleteByIds(table, ids, { ...opt, tx: this })
   }
 }
+
+function idComparator<T extends ObjectWithId>(a: T, b: T): number {
+  return a.id > b.id ? 1 : a.id < b.id ? -1 : 0
+}