@naturalcycles/datastore-lib 3.19.0 → 3.20.0

package/dist/datastore.db.d.ts

@@ -20,7 +20,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     protected KEY: symbol;
     ds(): Datastore;
     ping(): Promise<void>;
-    getByIds<ROW extends ObjectWithId>(table: string, ids: string[], _opt?: DatastoreDBOptions): Promise<ROW[]>;
+    getByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][], _opt?: DatastoreDBOptions): Promise<ROW[]>;
     getQueryKind(q: Query): string;
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBOptions): Promise<number>;
@@ -30,13 +30,13 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     /**
      * Returns saved entities with generated id/updated/created (non-mutating!)
      */
-    saveBatch<ROW extends ObjectWithId>(table: string, rows: ROW[], opt?: DatastoreDBSaveOptions<ROW>): Promise<void>;
+    saveBatch<ROW extends Partial<ObjectWithId>>(table: string, rows: ROW[], opt?: DatastoreDBSaveOptions<ROW>): Promise<void>;
     deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, opt?: DatastoreDBOptions): Promise<number>;
     /**
      * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
      * regardless if they were actually deleted or not.
      */
-    deleteByIds(table: string, ids: string[], opt?: DatastoreDBOptions): Promise<number>;
+    deleteByIds<ROW extends ObjectWithId>(table: string, ids: ROW['id'][], opt?: DatastoreDBOptions): Promise<number>;
     /**
      * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
      */
@@ -50,9 +50,9 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     getTableProperties(table: string): Promise<DatastorePropertyStats[]>;
     mapId<T = any>(o: any, preserveKey?: boolean): T;
     toDatastoreEntity<T = any>(kind: string, o: T & {
-        id?: string;
+        id?: string | number;
     }, excludeFromIndexes?: string[]): DatastorePayload<T>;
-    key(kind: string, id: string): Key;
+    key(kind: string, id: string | number): Key;
     getDsKey(o: any): Key | undefined;
     getKey(key: Key): string | undefined;
     getTables(): Promise<string[]>;

package/dist/datastore.db.js

@@ -13,6 +13,11 @@ const MAX_ITEMS = 500;
 const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN'];
 // Examples of errors:
 // UNKNOWN: Stream removed
+const methodMap = {
+    insert: 'insert',
+    update: 'update',
+    upsert: 'save',
+};
 /**
  * Datastore API:
  * https://googlecloudplatform.github.io/google-cloud-node/#/docs/datastore/1.0.3/datastore
@@ -79,6 +84,10 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                 const r = await (0, js_lib_1.pTimeout)(this.ds().get(keys), {
                     timeout: this.cfg.timeout,
                     name: `datastore.getByIds(${table}) second try`,
+                    errorData: {
+                        // This error will be grouped ACROSS all endpoints and usages
+                        fingerprint: ['DATASTORE_TIMEOUT'],
+                    },
                 });
                 rows = r[0];
             }
@@ -143,8 +152,9 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
      */
     async saveBatch(table, rows, opt = {}) {
         const entities = rows.map(obj => this.toDatastoreEntity(table, obj, opt.excludeFromIndexes));
+        const method = methodMap[opt.saveMethod || 'upsert'] || 'save';
         const save = (0, js_lib_1.pRetryFn)(async (batch) => {
-            await (opt.tx || this.ds()).save(batch);
+            await (opt.tx || this.ds())[method](batch);
         }, {
             // Here we retry the GOAWAY errors that are somewhat common for Datastore
             // Currently only retrying them here in .saveBatch(), cause probably they're only thrown when saving

package/dist/datastore.model.d.ts

@@ -88,7 +88,7 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
 export interface DatastoreDBOptions extends CommonDBOptions {
     tx?: Transaction;
 }
-export interface DatastoreDBSaveOptions<ROW extends ObjectWithId = AnyObjectWithId> extends CommonDBSaveOptions<ROW> {
+export interface DatastoreDBSaveOptions<ROW extends Partial<ObjectWithId> = AnyObjectWithId> extends CommonDBSaveOptions<ROW> {
     tx?: Transaction;
 }
 export interface DatastoreStats {

package/dist/index.d.ts

@@ -1,6 +1,5 @@
 import { DatastoreDB } from './datastore.db';
 import { DatastoreCredentials, DatastoreDBCfg, DatastoreDBOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePayload, DatastorePropertyStats, DatastoreStats, DatastoreType } from './datastore.model';
 import { DatastoreKeyValueDB, DatastoreKeyValueDBCfg } from './datastoreKeyValueDB';
-import { getDBAdapter } from './dbAdapter';
 export type { DatastorePayload, DatastoreDBCfg, DatastoreKeyValueDBCfg, DatastoreStats, DatastoreCredentials, DatastorePropertyStats, DatastoreDBStreamOptions, DatastoreDBOptions, DatastoreDBSaveOptions, };
-export { DatastoreDB, DatastoreType, getDBAdapter, DatastoreKeyValueDB };
+export { DatastoreDB, DatastoreType, DatastoreKeyValueDB };

package/dist/index.js

@@ -1,11 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DatastoreKeyValueDB = exports.getDBAdapter = exports.DatastoreType = exports.DatastoreDB = void 0;
+exports.DatastoreKeyValueDB = exports.DatastoreType = exports.DatastoreDB = void 0;
 const datastore_db_1 = require("./datastore.db");
 Object.defineProperty(exports, "DatastoreDB", { enumerable: true, get: function () { return datastore_db_1.DatastoreDB; } });
 const datastore_model_1 = require("./datastore.model");
 Object.defineProperty(exports, "DatastoreType", { enumerable: true, get: function () { return datastore_model_1.DatastoreType; } });
 const datastoreKeyValueDB_1 = require("./datastoreKeyValueDB");
 Object.defineProperty(exports, "DatastoreKeyValueDB", { enumerable: true, get: function () { return datastoreKeyValueDB_1.DatastoreKeyValueDB; } });
-const dbAdapter_1 = require("./dbAdapter");
-Object.defineProperty(exports, "getDBAdapter", { enumerable: true, get: function () { return dbAdapter_1.getDBAdapter; } });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.19.0",
+  "version": "3.20.0",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky install"

package/src/datastore.db.ts

@@ -3,6 +3,7 @@ import type { Datastore, Key, Query } from '@google-cloud/datastore'
 import {
   BaseCommonDB,
   CommonDB,
+  CommonDBSaveMethod,
   DBQuery,
   DBTransaction,
   mergeDBOperations,
@@ -48,6 +49,12 @@ const RETRY_ON = ['GOAWAY', 'UNAVAILABLE', 'UNKNOWN']
 // Examples of errors:
 // UNKNOWN: Stream removed
 
+const methodMap: Record<CommonDBSaveMethod, string> = {
+  insert: 'insert',
+  update: 'update',
+  upsert: 'save',
+}
+
 /**
  * Datastore API:
  * https://googlecloudplatform.github.io/google-cloud-node/#/docs/datastore/1.0.3/datastore
@@ -111,7 +118,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
   override async getByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
+    ids: ROW['id'][],
     _opt?: DatastoreDBOptions,
   ): Promise<ROW[]> {
     if (!ids.length) return []
@@ -138,6 +145,10 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         const r = await pTimeout(this.ds().get(keys), {
           timeout: this.cfg.timeout,
           name: `datastore.getByIds(${table}) second try`,
+          errorData: {
+            // This error will be grouped ACROSS all endpoints and usages
+            fingerprint: ['DATASTORE_TIMEOUT'],
+          },
         })
         rows = r[0]
       }
@@ -234,7 +245,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   /**
    * Returns saved entities with generated id/updated/created (non-mutating!)
    */
-  override async saveBatch<ROW extends ObjectWithId>(
+  override async saveBatch<ROW extends Partial<ObjectWithId>>(
     table: string,
     rows: ROW[],
     opt: DatastoreDBSaveOptions<ROW> = {},
@@ -243,9 +254,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       this.toDatastoreEntity(table, obj, opt.excludeFromIndexes as string[]),
     )
 
+    const method = methodMap[opt.saveMethod || 'upsert'] || 'save'
+
     const save = pRetryFn(
       async (batch: DatastorePayload<ROW>[]) => {
-        await (opt.tx || this.ds()).save(batch)
+        await (opt.tx || this.ds())[method](batch)
       },
       {
         // Here we retry the GOAWAY errors that are somewhat common for Datastore
@@ -298,9 +311,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
    * Limitation: Datastore's delete returns void, so we always return all ids here as "deleted"
    * regardless if they were actually deleted or not.
    */
-  override async deleteByIds(
+  override async deleteByIds<ROW extends ObjectWithId>(
     table: string,
-    ids: string[],
+    ids: ROW['id'][],
     opt: DatastoreDBOptions = {},
   ): Promise<number> {
     const keys = ids.map(id => this.key(table, id))
@@ -382,7 +395,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   // if key field exists on entity, it will be used as key (prevent to duplication of numeric keyed entities)
   toDatastoreEntity<T = any>(
     kind: string,
-    o: T & { id?: string },
+    o: T & { id?: string | number },
     excludeFromIndexes: string[] = [],
   ): DatastorePayload<T> {
     const key = this.getDsKey(o) || this.key(kind, o.id!)
@@ -397,7 +410,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  key(kind: string, id: string): Key {
+  key(kind: string, id: string | number): Key {
     _assert(id, `Cannot save "${kind}" entity without "id"`)
     return this.ds().key([kind, String(id)])
   }

package/src/datastore.model.ts

@@ -102,7 +102,7 @@ export interface DatastoreDBStreamOptions extends DatastoreDBOptions {
 export interface DatastoreDBOptions extends CommonDBOptions {
   tx?: Transaction
 }
-export interface DatastoreDBSaveOptions<ROW extends ObjectWithId = AnyObjectWithId>
+export interface DatastoreDBSaveOptions<ROW extends Partial<ObjectWithId> = AnyObjectWithId>
   extends CommonDBSaveOptions<ROW> {
   tx?: Transaction
 }

package/src/datastoreKeyValueDB.ts

@@ -50,7 +50,7 @@ export class DatastoreKeyValueDB implements CommonKeyValueDB {
       .limit(limit || 0)
 
     return this.db.streamQuery<KVObject>(q).pipe(
-      transformMapSimple<ObjectWithId, string>(objectWithId => objectWithId.id, {
+      transformMapSimple<ObjectWithId<string>, string>(objectWithId => objectWithId.id, {
         errorMode: ErrorMode.SUPPRESS, // cause .pipe() cannot propagate errors
       }),
     )

package/src/index.ts

@@ -11,7 +11,6 @@ import {
   DatastoreType,
 } from './datastore.model'
 import { DatastoreKeyValueDB, DatastoreKeyValueDBCfg } from './datastoreKeyValueDB'
-import { getDBAdapter } from './dbAdapter'
 
 export type {
   DatastorePayload,
@@ -25,4 +24,4 @@ export type {
   DatastoreDBSaveOptions,
 }
 
-export { DatastoreDB, DatastoreType, getDBAdapter, DatastoreKeyValueDB }
+export { DatastoreDB, DatastoreType, DatastoreKeyValueDB }

package/dist/dbAdapter.d.ts

@@ -1,2 +0,0 @@
-import { DatastoreDB } from './datastore.db';
-export declare function getDBAdapter(cfgStr?: string): DatastoreDB;

package/dist/dbAdapter.js

@@ -1,13 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDBAdapter = void 0;
-const datastore_db_1 = require("./datastore.db");
-function getDBAdapter(cfgStr) {
-    const cfg = cfgStr
-        ? JSON.parse(cfgStr)
-        : {
-            useLegacyGRPC: true,
-        };
-    return new datastore_db_1.DatastoreDB(cfg);
-}
-exports.getDBAdapter = getDBAdapter;

package/src/dbAdapter.ts

@@ -1,11 +0,0 @@
-import { DatastoreDB } from './datastore.db'
-import { DatastoreDBCfg } from './datastore.model'
-
-export function getDBAdapter(cfgStr?: string): DatastoreDB {
-  const cfg: DatastoreDBCfg = cfgStr
-    ? JSON.parse(cfgStr)
-    : {
-        useLegacyGRPC: true,
-      }
-  return new DatastoreDB(cfg)
-}