@naturalcycles/datastore-lib 3.39.4 → 4.0.1

package/dist/DatastoreStreamReadable.d.ts

@@ -2,7 +2,7 @@ import { Readable } from 'node:stream';
 import type { Query } from '@google-cloud/datastore';
 import type { CommonLogger } from '@naturalcycles/js-lib';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import type { DatastoreDBStreamOptions } from './datastore.model';
+import type { DatastoreDBStreamOptions } from './datastore.model.js';
 export declare class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
     private q;
     private logger;

package/dist/DatastoreStreamReadable.js

@@ -1,26 +1,27 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DatastoreStreamReadable = void 0;
-const node_stream_1 = require("node:stream");
-const js_lib_1 = require("@naturalcycles/js-lib");
-class DatastoreStreamReadable extends node_stream_1.Readable {
+import { Readable } from 'node:stream';
+import { _ms, pRetry } from '@naturalcycles/js-lib';
+export class DatastoreStreamReadable extends Readable {
+    q;
+    logger;
+    originalLimit;
+    rowsRetrieved = 0;
+    endCursor;
+    running = false;
+    done = false;
+    lastQueryDone;
+    totalWait = 0;
+    table;
+    /**
+     * Used to support maxWait
+     */
+    lastReadTimestamp = 0;
+    maxWaitInterval;
+    opt;
+    dsOpt;
     constructor(q, opt, logger) {
         super({ objectMode: true });
         this.q = q;
         this.logger = logger;
-        this.rowsRetrieved = 0;
-        this.running = false;
-        this.done = false;
-        this.totalWait = 0;
-        /**
-         * Used to support maxWait
-         */
-        this.lastReadTimestamp = 0;
-        /**
-         * Counts how many times _read was called.
-         * For debugging.
-         */
-        this.count = 0;
         this.opt = {
             rssLimitMB: 1000,
             batchSize: 1000,
@@ -75,7 +76,7 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
         let rows = [];
         let info = {};
         try {
-            await (0, js_lib_1.pRetry)(async () => {
+            await pRetry(async () => {
                 const res = await q.run(this.dsOpt);
                 rows = res[0];
                 info = res[1];
@@ -98,9 +99,9 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
             return;
         }
         this.rowsRetrieved += rows.length;
-        this.logger.log(`${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
+        this.logger.log(`${this.table} got ${rows.length} rows, ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`, info.moreResults);
         if (!rows.length) {
-            this.logger.warn(`${this.table} got 0 rows, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`, info.moreResults);
+            this.logger.warn(`${this.table} got 0 rows, totalWait: ${_ms(this.totalWait)}`, info.moreResults);
         }
         this.endCursor = info.endCursor;
         this.running = false; // ready to take more _reads
@@ -109,7 +110,7 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
         if (!info.endCursor ||
             info.moreResults === 'NO_MORE_RESULTS' ||
             (this.originalLimit && this.rowsRetrieved >= this.originalLimit)) {
-            this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${(0, js_lib_1._ms)(this.totalWait)}`);
+            this.logger.log(`!!!! DONE! ${this.rowsRetrieved} rowsRetrieved, totalWait: ${_ms(this.totalWait)}`);
             this.push(null);
             this.done = true;
             clearInterval(this.maxWaitInterval);
@@ -128,6 +129,11 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
             }
         }
     }
+    /**
+     * Counts how many times _read was called.
+     * For debugging.
+     */
+    count = 0;
     _read() {
         this.lastReadTimestamp = Date.now();
         // console.log(`_read called ${++this.count}, wasRunning: ${this.running}`) // debugging
@@ -147,4 +153,3 @@ class DatastoreStreamReadable extends node_stream_1.Readable {
         }
     }
 }
-exports.DatastoreStreamReadable = DatastoreStreamReadable;

package/dist/datastore.db.d.ts

@@ -3,7 +3,7 @@ import type { CommonDB, CommonDBOptions, CommonDBReadOptions, CommonDBSaveOption
 import { BaseCommonDB } from '@naturalcycles/db-lib';
 import type { CommonLogger, JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import type { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBReadOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePropertyStats, DatastoreStats } from './datastore.model';
+import type { DatastoreDBCfg, DatastoreDBOptions, DatastoreDBReadOptions, DatastoreDBSaveOptions, DatastoreDBStreamOptions, DatastorePropertyStats, DatastoreStats } from './datastore.model.js';
 /**
  * Datastore API:
  * https://googlecloudplatform.github.io/google-cloud-node/#/docs/datastore/1.0.3/datastore
@@ -20,7 +20,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
      * Datastore.KEY
      */
     protected KEY: symbol;
-    ds(): Datastore;
+    ds(): Promise<Datastore>;
     private getPropertyFilter;
     private getDatastoreLib;
     ping(): Promise<void>;
@@ -28,8 +28,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     runQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<RunQueryResult<ROW>>;
     runQueryCount<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBReadOptions): Promise<number>;
     private runDatastoreQuery;
-    private runQueryStream;
-    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, opt?: DatastoreDBStreamOptions): ReadableTyped<ROW>;
+    streamQuery<ROW extends ObjectWithId>(dbQuery: DBQuery<ROW>, _opt?: DatastoreDBStreamOptions): ReadableTyped<ROW>;
     /**
      * Returns saved entities with generated id/updated/created (non-mutating!)
      */
@@ -50,7 +49,7 @@ export declare class DatastoreDB extends BaseCommonDB implements CommonDB {
     getTableProperties(table: string): Promise<DatastorePropertyStats[]>;
     private mapId;
     private toDatastoreEntity;
-    key(kind: string, id: string): Key;
+    key(ds: Datastore, kind: string, id: string): Key;
     private getDsKey;
     getKey(key: Key): string | undefined;
     createTable<ROW extends ObjectWithId>(_table: string, _schema: JsonSchemaObject<ROW>): Promise<void>;

package/dist/datastore.db.js

@@ -1,12 +1,10 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DatastoreDBTransaction = exports.DatastoreDB = void 0;
-const db_lib_1 = require("@naturalcycles/db-lib");
-const js_lib_1 = require("@naturalcycles/js-lib");
-const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
-const datastore_model_1 = require("./datastore.model");
-const DatastoreStreamReadable_1 = require("./DatastoreStreamReadable");
-const query_util_1 = require("./query.util");
+import { Transform } from 'node:stream';
+import { BaseCommonDB, commonDBFullSupport } from '@naturalcycles/db-lib';
+import { _assert, _chunk, _errorDataAppend, _omit, commonLoggerMinLevel, pMap, pRetry, pRetryFn, pTimeout, TimeoutError, } from '@naturalcycles/js-lib';
+import { boldWhite } from '@naturalcycles/nodejs-lib';
+import { DatastoreType } from './datastore.model.js';
+import { DatastoreStreamReadable } from './DatastoreStreamReadable.js';
+import { dbQueryToDatastoreQuery } from './query.util.js';
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500;
 // It's an empyrical value, but anything less than infinity is better than infinity
@@ -32,27 +30,33 @@ const methodMap = {
  * https://googlecloudplatform.github.io/google-cloud-node/#/docs/datastore/1.0.3/datastore
  * https://cloud.google.com/datastore/docs/datastore-api-tutorial
  */
-class DatastoreDB extends db_lib_1.BaseCommonDB {
+export class DatastoreDB extends BaseCommonDB {
+    support = {
+        ...commonDBFullSupport,
+        patchByQuery: false,
+        increment: false,
+    };
     constructor(cfg = {}) {
         super();
-        this.support = {
-            ...db_lib_1.commonDBFullSupport,
-            patchByQuery: false,
-            increment: false,
-        };
         this.cfg = {
             logger: console,
             ...cfg,
         };
     }
+    cfg;
+    cachedDatastore;
+    /**
+     * Datastore.KEY
+     */
+    KEY;
     // @memo() // not used to be able to connect to many DBs in the same server instance
-    ds() {
+    async ds() {
         if (!this.cachedDatastore) {
-            (0, js_lib_1._assert)(process.env['APP_ENV'] !== 'test', 'DatastoreDB cannot be used in Test env, please use InMemoryDB');
-            const DS = this.getDatastoreLib().Datastore;
+            _assert(process.env['APP_ENV'] !== 'test', 'DatastoreDB cannot be used in Test env, please use InMemoryDB');
+            const DS = (await this.getDatastoreLib()).Datastore;
             this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT'];
             if (this.cfg.projectId) {
-                this.cfg.logger.log(`DatastoreDB connected to ${(0, nodejs_lib_1.boldWhite)(this.cfg.projectId)}`);
+                this.cfg.logger.log(`DatastoreDB connected to ${boldWhite(this.cfg.projectId)}`);
             }
             else if (process.env['GOOGLE_APPLICATION_CREDENTIALS']) {
                 this.cfg.logger.log(`DatastoreDB connected via GOOGLE_APPLICATION_CREDENTIALS`);
@@ -65,12 +69,13 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         }
         return this.cachedDatastore;
     }
-    getPropertyFilter() {
-        return this.getDatastoreLib().PropertyFilter;
+    async getPropertyFilter() {
+        return (await this.getDatastoreLib()).PropertyFilter;
     }
-    getDatastoreLib() {
+    async getDatastoreLib() {
         // Lazy-loading
-        return require('@google-cloud/datastore');
+        const lib = await import('@google-cloud/datastore');
+        return lib;
     }
     async ping() {
         await this.getAllStats();
@@ -78,31 +83,32 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
     async getByIds(table, ids, opt = {}) {
         if (!ids.length)
             return [];
-        const keys = ids.map(id => this.key(table, id));
+        let ds = await this.ds();
+        const keys = ids.map(id => this.key(ds, table, id));
         let rows;
         const dsOpt = this.getRunQueryOptions(opt);
         if (this.cfg.timeout) {
             // First try
             try {
-                const r = await (0, js_lib_1.pTimeout)(() => (opt.tx?.tx || this.ds()).get(keys, dsOpt), {
+                const r = await pTimeout(() => (opt.tx?.tx || ds).get(keys, dsOpt), {
                     timeout: this.cfg.timeout,
                     name: `datastore.getByIds(${table})`,
                 });
                 rows = r[0];
             }
             catch (err) {
-                if (!(err instanceof js_lib_1.TimeoutError)) {
+                if (!(err instanceof TimeoutError)) {
                     // Not a timeout error, re-throw
                     throw err;
                 }
                 this.cfg.logger.log('datastore recreated on error');
                 // This is to debug "GCP Datastore Timeout issue"
-                const datastoreLib = require('@google-cloud/datastore');
+                const datastoreLib = await this.getDatastoreLib();
                 const DS = datastoreLib.Datastore;
-                this.cachedDatastore = new DS(this.cfg);
+                ds = this.cachedDatastore = new DS(this.cfg);
                 // Second try (will throw)
                 try {
-                    const r = await (0, js_lib_1.pRetry)(() => (opt.tx?.tx || this.ds()).get(keys, dsOpt), {
+                    const r = await pRetry(() => (opt.tx?.tx || ds).get(keys, dsOpt), {
                         ...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
                         maxAttempts: 3,
                         timeout: this.cfg.timeout,
@@ -110,8 +116,8 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                     rows = r[0];
                 }
                 catch (err) {
-                    if (err instanceof js_lib_1.TimeoutError) {
-                        (0, js_lib_1._errorDataAppend)(err, {
+                    if (err instanceof TimeoutError) {
+                        _errorDataAppend(err, {
                             fingerprint: [DATASTORE_TIMEOUT],
                         });
                     }
@@ -120,8 +126,8 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             }
         }
         else {
-            rows = await (0, js_lib_1.pRetry)(async () => {
-                return (await this.ds().get(keys, dsOpt))[0];
+            rows = await pRetry(async () => {
+                return (await ds.get(keys, dsOpt))[0];
             }, this.getPRetryOptions(`datastore.getByIds(${table})`));
         }
         return (rows
@@ -142,69 +148,80 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                 rows: await this.getByIds(dbQuery.table, ids, opt),
             };
         }
-        const q = (0, query_util_1.dbQueryToDatastoreQuery)(dbQuery, this.ds().createQuery(dbQuery.table), this.getPropertyFilter());
+        const ds = await this.ds();
+        const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table), await this.getPropertyFilter());
         const dsOpt = this.getRunQueryOptions(opt);
         const qr = await this.runDatastoreQuery(q, dsOpt);
         // Special case when projection query didn't specify 'id'
         if (dbQuery._selectedFieldNames && !dbQuery._selectedFieldNames.includes('id')) {
-            qr.rows = qr.rows.map(r => (0, js_lib_1._omit)(r, ['id']));
+            qr.rows = qr.rows.map(r => _omit(r, ['id']));
         }
         return qr;
     }
     async runQueryCount(dbQuery, opt = {}) {
-        const q = (0, query_util_1.dbQueryToDatastoreQuery)(dbQuery.select([]), this.ds().createQuery(dbQuery.table), this.getPropertyFilter());
-        const aq = this.ds().createAggregationQuery(q).count('count');
+        const ds = await this.ds();
+        const q = dbQueryToDatastoreQuery(dbQuery.select([]), ds.createQuery(dbQuery.table), await this.getPropertyFilter());
+        const aq = ds.createAggregationQuery(q).count('count');
         const dsOpt = this.getRunQueryOptions(opt);
-        const [entities] = await this.ds().runAggregationQuery(aq, dsOpt);
+        const [entities] = await ds.runAggregationQuery(aq, dsOpt);
         return entities[0]?.count;
     }
     async runDatastoreQuery(q, dsOpt) {
-        const [entities, queryResult] = await this.ds().runQuery(q, dsOpt);
+        const ds = await this.ds();
+        const [entities, queryResult] = await ds.runQuery(q, dsOpt);
         const rows = entities.map(e => this.mapId(e));
         return {
             ...queryResult,
             rows,
         };
     }
-    runQueryStream(q, _opt) {
-        const opt = {
-            ...this.cfg.streamOptions,
-            ..._opt,
-        };
-        const dsOpt = this.getRunQueryOptions(opt);
-        return (opt.experimentalCursorStream
-            ? new DatastoreStreamReadable_1.DatastoreStreamReadable(q, opt, (0, js_lib_1.commonLoggerMinLevel)(this.cfg.logger, opt.debug ? 'log' : 'warn'))
-            : this.ds().runQueryStream(q, dsOpt)).map(chunk => this.mapId(chunk));
-    }
-    streamQuery(dbQuery, opt) {
-        const q = (0, query_util_1.dbQueryToDatastoreQuery)(dbQuery, this.ds().createQuery(dbQuery.table), this.getPropertyFilter());
-        return this.runQueryStream(q, opt);
+    streamQuery(dbQuery, _opt) {
+        const transform = new Transform({
+            objectMode: true,
+            transform: (chunk, _, cb) => {
+                cb(null, this.mapId(chunk));
+            },
+        });
+        void this.ds().then(async (ds) => {
+            const q = dbQueryToDatastoreQuery(dbQuery, ds.createQuery(dbQuery.table), await this.getPropertyFilter());
+            const opt = {
+                ...this.cfg.streamOptions,
+                ..._opt,
+            };
+            (opt.experimentalCursorStream
+                ? new DatastoreStreamReadable(q, opt, commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'))
+                : ds.runQueryStream(q, this.getRunQueryOptions(opt)))
+                .on('error', err => transform.emit('error', err))
+                .pipe(transform);
+        });
+        return transform;
     }
     // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js
     /**
      * Returns saved entities with generated id/updated/created (non-mutating!)
      */
     async saveBatch(table, rows, opt = {}) {
-        const entities = rows.map(obj => this.toDatastoreEntity(table, obj, opt.excludeFromIndexes));
+        const ds = await this.ds();
+        const entities = rows.map(obj => this.toDatastoreEntity(ds, table, obj, opt.excludeFromIndexes));
         const method = methodMap[opt.saveMethod || 'upsert'] || 'save';
-        const save = (0, js_lib_1.pRetryFn)(async (batch) => {
-            await (opt.tx?.tx || this.ds())[method](batch);
+        const save = pRetryFn(async (batch) => {
+            await (opt.tx?.tx || ds)[method](batch);
         }, this.getPRetryOptions(`DatastoreLib.saveBatch(${table})`));
         try {
-            const chunks = (0, js_lib_1._chunk)(entities, MAX_ITEMS);
+            const chunks = _chunk(entities, MAX_ITEMS);
             if (chunks.length === 1) {
                 // Not using pMap in hope to preserve stack trace
                 await save(chunks[0]);
             }
             else {
-                await (0, js_lib_1.pMap)(chunks, async (batch) => await save(batch), {
+                await pMap(chunks, async (batch) => await save(batch), {
                     concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
                 });
             }
         }
         catch (err) {
-            if (err instanceof js_lib_1.TimeoutError) {
-                (0, js_lib_1._errorDataAppend)(err, {
+            if (err instanceof TimeoutError) {
+                _errorDataAppend(err, {
                     fingerprint: [DATASTORE_TIMEOUT],
                 });
             }
@@ -219,7 +236,8 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             const ids = idFilter.op === '==' ? [idFilter.val] : idFilter.val;
             return await this.deleteByIds(q.table, ids, opt);
         }
-        const datastoreQuery = (0, query_util_1.dbQueryToDatastoreQuery)(q.select([]), this.ds().createQuery(q.table), this.getPropertyFilter());
+        const ds = await this.ds();
+        const datastoreQuery = dbQueryToDatastoreQuery(q.select([]), ds.createQuery(q.table), await this.getPropertyFilter());
         const dsOpt = this.getRunQueryOptions(opt);
         const { rows } = await this.runDatastoreQuery(datastoreQuery, dsOpt);
         return await this.deleteByIds(q.table, rows.map(obj => obj.id), opt);
@@ -229,15 +247,17 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
      * regardless if they were actually deleted or not.
      */
     async deleteByIds(table, ids, opt = {}) {
-        const keys = ids.map(id => this.key(table, id));
-        await (0, js_lib_1.pMap)((0, js_lib_1._chunk)(keys, MAX_ITEMS), async (batch) => await (opt.tx?.tx || this.ds()).delete(batch), {
+        const ds = await this.ds();
+        const keys = ids.map(id => this.key(ds, table, id));
+        await pMap(_chunk(keys, MAX_ITEMS), async (batch) => await (opt.tx?.tx || ds).delete(batch), {
             concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
         });
         return ids.length;
     }
     async runInTransaction(fn, opt = {}) {
+        const ds = await this.ds();
         const { readOnly } = opt;
-        const datastoreTx = this.ds().transaction({
+        const datastoreTx = ds.transaction({
             readOnly,
         });
         try {
@@ -252,21 +272,23 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         }
     }
     async getAllStats() {
-        const q = this.ds().createQuery('__Stat_Kind__');
-        const [statsArray] = await this.ds().runQuery(q);
+        const ds = await this.ds();
+        const q = ds.createQuery('__Stat_Kind__');
+        const [statsArray] = await ds.runQuery(q);
         return statsArray || [];
     }
     /**
      * Returns undefined e.g when Table is non-existing
      */
     async getStats(table) {
-        const { PropertyFilter } = this.getDatastoreLib();
-        const q = this.ds()
+        const ds = await this.ds();
+        const propertyFilter = await this.getPropertyFilter();
+        const q = ds
             .createQuery('__Stat_Kind__')
             // .filter('kind_name', table)
-            .filter(new PropertyFilter('kind_name', '=', table))
+            .filter(new propertyFilter('kind_name', '=', table))
             .limit(1);
-        const [statsArray] = await this.ds().runQuery(q);
+        const [statsArray] = await ds.runQuery(q);
         const [stats] = statsArray;
         return stats;
     }
@@ -275,11 +297,12 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         return stats?.count;
     }
     async getTableProperties(table) {
-        const q = this.ds()
+        const ds = await this.ds();
+        const q = ds
             .createQuery('__Stat_PropertyType_PropertyName_Kind__')
             // .filter('kind_name', table)
-            .filter(new (this.getPropertyFilter())('kind_name', '=', table));
-        const [stats] = await this.ds().runQuery(q);
+            .filter(new (await this.getPropertyFilter())('kind_name', '=', table));
+        const [stats] = await ds.runQuery(q);
         return stats;
     }
     mapId(o, preserveKey = false) {
@@ -294,8 +317,8 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         return r;
     }
     // if key field exists on entity, it will be used as key (prevent to duplication of numeric keyed entities)
-    toDatastoreEntity(kind, o, excludeFromIndexes = []) {
-        const key = this.getDsKey(o) || this.key(kind, o.id);
+    toDatastoreEntity(ds, kind, o, excludeFromIndexes = []) {
+        const key = this.getDsKey(o) || this.key(ds, kind, o.id);
         const data = Object.assign({}, o);
         delete data.id;
         delete data[this.KEY];
@@ -305,9 +328,9 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             excludeFromIndexes,
         };
     }
-    key(kind, id) {
-        (0, js_lib_1._assert)(id, `Cannot save "${kind}" entity without "id"`);
-        return this.ds().key([kind, id]);
+    key(ds, kind, id) {
+        _assert(id, `Cannot save "${kind}" entity without "id"`);
+        return ds.key([kind, id]);
     }
     getDsKey(o) {
         return o?.[this.KEY];
@@ -338,17 +361,17 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
             .forEach(stats => {
             const { property_type: dtype } = stats;
             const name = stats.property_name;
-            if (dtype === datastore_model_1.DatastoreType.Blob) {
+            if (dtype === DatastoreType.Blob) {
                 s.properties[name] = {
                     instanceof: 'Buffer',
                 };
             }
-            else if (dtype === datastore_model_1.DatastoreType.Text || dtype === datastore_model_1.DatastoreType.String) {
+            else if (dtype === DatastoreType.Text || dtype === DatastoreType.String) {
                 s.properties[name] = {
                     type: 'string',
                 };
             }
-            else if (dtype === datastore_model_1.DatastoreType.EmbeddedEntity) {
+            else if (dtype === DatastoreType.EmbeddedEntity) {
                 s.properties[name] = {
                     type: 'object',
                     additionalProperties: true,
@@ -356,26 +379,26 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
                     required: [],
                 };
             }
-            else if (dtype === datastore_model_1.DatastoreType.Integer) {
+            else if (dtype === DatastoreType.Integer) {
                 s.properties[name] = {
                     type: 'integer',
                 };
             }
-            else if (dtype === datastore_model_1.DatastoreType.Float) {
+            else if (dtype === DatastoreType.Float) {
                 s.properties[name] = {
                     type: 'number',
                 };
             }
-            else if (dtype === datastore_model_1.DatastoreType.Boolean) {
+            else if (dtype === DatastoreType.Boolean) {
                 s.properties[name] = {
                     type: 'boolean',
                 };
             }
-            else if (dtype === datastore_model_1.DatastoreType.DATE_TIME) {
+            else if (dtype === DatastoreType.DATE_TIME) {
                 // Don't know how to map it properly
                 s.properties[name] = {};
             }
-            else if (dtype === datastore_model_1.DatastoreType.NULL) {
+            else if (dtype === DatastoreType.NULL) {
                 // check, maybe we can just skip this type and do nothing?
                 s.properties[name] ||= {
                     type: 'null',
@@ -427,11 +450,12 @@ class DatastoreDB extends db_lib_1.BaseCommonDB {
         };
     }
 }
-exports.DatastoreDB = DatastoreDB;
 /**
  * https://cloud.google.com/datastore/docs/concepts/transactions#datastore-datastore-transactional-update-nodejs
  */
-class DatastoreDBTransaction {
+export class DatastoreDBTransaction {
+    db;
+    tx;
     constructor(db, tx) {
         this.db = db;
         this.tx = tx;
@@ -455,4 +479,3 @@ class DatastoreDBTransaction {
         return await this.db.deleteByIds(table, ids, { ...opt, tx: this });
     }
 }
-exports.DatastoreDBTransaction = DatastoreDBTransaction;

package/dist/datastore.model.js

@@ -1,7 +1,4 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DatastoreType = void 0;
-var DatastoreType;
+export var DatastoreType;
 (function (DatastoreType) {
     DatastoreType["Blob"] = "Blob";
     DatastoreType["Text"] = "Text";
@@ -12,4 +9,4 @@ var DatastoreType;
     DatastoreType["DATE_TIME"] = "Date/Time";
     DatastoreType["Boolean"] = "Boolean";
     DatastoreType["NULL"] = "NULL";
-})(DatastoreType || (exports.DatastoreType = DatastoreType = {}));
+})(DatastoreType || (DatastoreType = {}));

package/dist/datastoreKeyValueDB.d.ts

@@ -1,7 +1,7 @@
 import type { CommonKeyValueDB, IncrementTuple, KeyValueDBTuple } from '@naturalcycles/db-lib';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { DatastoreDB } from './datastore.db';
-import type { DatastoreDBCfg } from './datastore.model';
+import { DatastoreDB } from './datastore.db.js';
+import type { DatastoreDBCfg } from './datastore.model.js';
 export interface DatastoreKeyValueDBCfg extends DatastoreDBCfg {
 }
 export declare class DatastoreKeyValueDB implements CommonKeyValueDB {

package/dist/datastoreKeyValueDB.js

@@ -1,19 +1,18 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DatastoreKeyValueDB = void 0;
-const db_lib_1 = require("@naturalcycles/db-lib");
-const js_lib_1 = require("@naturalcycles/js-lib");
-const datastore_db_1 = require("./datastore.db");
+import { commonKeyValueDBFullSupport, DBQuery } from '@naturalcycles/db-lib';
+import { AppError } from '@naturalcycles/js-lib';
+import { DatastoreDB } from './datastore.db.js';
 const excludeFromIndexes = ['v'];
-class DatastoreKeyValueDB {
+export class DatastoreKeyValueDB {
+    cfg;
     constructor(cfg) {
         this.cfg = cfg;
-        this.support = {
-            ...db_lib_1.commonKeyValueDBFullSupport,
-            increment: false,
-        };
-        this.db = new datastore_db_1.DatastoreDB(cfg);
+        this.db = new DatastoreDB(cfg);
     }
+    db;
+    support = {
+        ...commonKeyValueDBFullSupport,
+        increment: false,
+    };
     async ping() {
         await this.db.ping();
     }
@@ -30,7 +29,7 @@ class DatastoreKeyValueDB {
         });
     }
     streamIds(table, limit) {
-        const q = db_lib_1.DBQuery.create(table)
+        const q = DBQuery.create(table)
             .select(['id'])
             .limit(limit || 0);
         return (this.db
@@ -40,25 +39,24 @@ class DatastoreKeyValueDB {
     }
     streamValues(table, limit) {
         // `select v` doesn't work for some reason
-        const q = db_lib_1.DBQuery.create(table).limit(limit || 0);
+        const q = DBQuery.create(table).limit(limit || 0);
         return (this.db
             .streamQuery(q)
             // .on('error', err => stream.emit('error', err))
             .map(r => r.v));
     }
     streamEntries(table, limit) {
-        const q = db_lib_1.DBQuery.create(table).limit(limit || 0);
+        const q = DBQuery.create(table).limit(limit || 0);
         return (this.db
             .streamQuery(q)
             // .on('error', err => stream.emit('error', err))
             .map(r => [r.id, r.v]));
     }
     async count(table) {
-        const q = db_lib_1.DBQuery.create(table);
+        const q = DBQuery.create(table);
         return await this.db.runQueryCount(q);
     }
     async incrementBatch(_table, _entries) {
-        throw new js_lib_1.AppError('DatastoreKeyValueDB.incrementBatch() is not implemented');
+        throw new AppError('DatastoreKeyValueDB.incrementBatch() is not implemented');
     }
 }
-exports.DatastoreKeyValueDB = DatastoreKeyValueDB;

package/dist/index.d.ts

@@ -1,3 +1,3 @@
-export * from './datastore.db';
-export * from './datastore.model';
-export * from './datastoreKeyValueDB';
+export * from './datastore.db.js';
+export * from './datastore.model.js';
+export * from './datastoreKeyValueDB.js';

package/dist/index.js

@@ -1,6 +1,3 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const tslib_1 = require("tslib");
-tslib_1.__exportStar(require("./datastore.db"), exports);
-tslib_1.__exportStar(require("./datastore.model"), exports);
-tslib_1.__exportStar(require("./datastoreKeyValueDB"), exports);
+export * from './datastore.db.js';
+export * from './datastore.model.js';
+export * from './datastoreKeyValueDB.js';

package/dist/query.util.js

@@ -1,6 +1,3 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.dbQueryToDatastoreQuery = dbQueryToDatastoreQuery;
 const FNAME_MAP = {
     id: '__key__',
 };
@@ -9,7 +6,7 @@ const OP_MAP = {
     in: 'IN',
     'not-in': 'NOT_IN',
 };
-function dbQueryToDatastoreQuery(dbQuery, emptyQuery, propertyFilterClass) {
+export function dbQueryToDatastoreQuery(dbQuery, emptyQuery, propertyFilterClass) {
     let q = emptyQuery;
     // filter
     for (const f of dbQuery._filters) {

package/package.json

@@ -1,6 +1,7 @@
 {
   "name": "@naturalcycles/datastore-lib",
-  "version": "3.39.4",
+  "type": "module",
+  "version": "4.0.1",
   "description": "Opinionated library to work with Google Datastore",
   "scripts": {
     "prepare": "husky",

package/src/DatastoreStreamReadable.ts

@@ -1,10 +1,10 @@
 import { Readable } from 'node:stream'
 import type { Query } from '@google-cloud/datastore'
-import type { RunQueryInfo, RunQueryOptions } from '@google-cloud/datastore/build/src/query'
+import type { RunQueryInfo, RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
 import type { CommonLogger, UnixTimestampMillis } from '@naturalcycles/js-lib'
 import { _ms, pRetry } from '@naturalcycles/js-lib'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import type { DatastoreDBStreamOptions } from './datastore.model'
+import type { DatastoreDBStreamOptions } from './datastore.model.js'
 
 export class DatastoreStreamReadable<T = any> extends Readable implements ReadableTyped<T> {
   private readonly originalLimit: number

package/src/datastore.db.ts

@@ -1,5 +1,6 @@
+import { Transform } from 'node:stream'
 import type { Datastore, Key, PropertyFilter, Query, Transaction } from '@google-cloud/datastore'
-import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query'
+import type { RunQueryOptions } from '@google-cloud/datastore/build/src/query.js'
 import type {
   CommonDB,
   CommonDBOptions,
@@ -49,10 +50,10 @@ import type {
   DatastorePayload,
   DatastorePropertyStats,
   DatastoreStats,
-} from './datastore.model'
-import { DatastoreType } from './datastore.model'
-import { DatastoreStreamReadable } from './DatastoreStreamReadable'
-import { dbQueryToDatastoreQuery } from './query.util'
+} from './datastore.model.js'
+import { DatastoreType } from './datastore.model.js'
+import { DatastoreStreamReadable } from './DatastoreStreamReadable.js'
+import { dbQueryToDatastoreQuery } from './query.util.js'
 
 // Datastore (also Firestore and other Google APIs) supports max 500 of items when saving/deleting, etc.
 const MAX_ITEMS = 500
@@ -108,14 +109,14 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   protected KEY!: symbol
 
   // @memo() // not used to be able to connect to many DBs in the same server instance
-  ds(): Datastore {
+  async ds(): Promise<Datastore> {
     if (!this.cachedDatastore) {
       _assert(
         process.env['APP_ENV'] !== 'test',
         'DatastoreDB cannot be used in Test env, please use InMemoryDB',
       )
 
-      const DS = this.getDatastoreLib().Datastore as typeof Datastore
+      const DS = (await this.getDatastoreLib()).Datastore as typeof Datastore
       this.cfg.projectId ||= this.cfg.credentials?.project_id || process.env['GOOGLE_CLOUD_PROJECT']
 
       if (this.cfg.projectId) {
@@ -135,13 +136,14 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     return this.cachedDatastore
   }
 
-  private getPropertyFilter(): typeof PropertyFilter {
-    return this.getDatastoreLib().PropertyFilter
+  private async getPropertyFilter(): Promise<typeof PropertyFilter> {
+    return (await this.getDatastoreLib()).PropertyFilter
   }
 
-  private getDatastoreLib(): any {
+  private async getDatastoreLib(): Promise<any> {
     // Lazy-loading
-    return require('@google-cloud/datastore')
+    const lib = await import('@google-cloud/datastore')
+    return lib
   }
 
   override async ping(): Promise<void> {
@@ -154,7 +156,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     opt: DatastoreDBReadOptions = {},
   ): Promise<ROW[]> {
     if (!ids.length) return []
-    const keys = ids.map(id => this.key(table, id))
+    let ds = await this.ds()
+    const keys = ids.map(id => this.key(ds, table, id))
     let rows: any[]
 
     const dsOpt = this.getRunQueryOptions(opt)
@@ -163,7 +166,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       // First try
       try {
         const r = await pTimeout(
-          () => ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).get(keys, dsOpt),
+          () => ((opt.tx as DatastoreDBTransaction)?.tx || ds).get(keys, dsOpt),
           {
             timeout: this.cfg.timeout,
             name: `datastore.getByIds(${table})`,
@@ -179,14 +182,14 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
         this.cfg.logger.log('datastore recreated on error')
 
         // This is to debug "GCP Datastore Timeout issue"
-        const datastoreLib = require('@google-cloud/datastore')
+        const datastoreLib = await this.getDatastoreLib()
         const DS = datastoreLib.Datastore as typeof Datastore
-        this.cachedDatastore = new DS(this.cfg)
+        ds = this.cachedDatastore = new DS(this.cfg)
 
         // Second try (will throw)
         try {
           const r = await pRetry(
-            () => ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).get(keys, dsOpt),
+            () => ((opt.tx as DatastoreDBTransaction)?.tx || ds).get(keys, dsOpt),
             {
               ...this.getPRetryOptions(`datastore.getByIds(${table}) second try`),
               maxAttempts: 3,
@@ -206,7 +209,7 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     } else {
       rows = await pRetry(
         async () => {
-          return (await this.ds().get(keys, dsOpt))[0]
+          return (await ds.get(keys, dsOpt))[0]
         },
         this.getPRetryOptions(`datastore.getByIds(${table})`),
       )
@@ -239,10 +242,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       }
     }
 
+    const ds = await this.ds()
     const q = dbQueryToDatastoreQuery(
       dbQuery,
-      this.ds().createQuery(dbQuery.table),
-      this.getPropertyFilter(),
+      ds.createQuery(dbQuery.table),
+      await this.getPropertyFilter(),
     )
     const dsOpt = this.getRunQueryOptions(opt)
     const qr = await this.runDatastoreQuery<ROW>(q, dsOpt)
@@ -259,14 +263,15 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     dbQuery: DBQuery<ROW>,
     opt: DatastoreDBReadOptions = {},
   ): Promise<number> {
+    const ds = await this.ds()
     const q = dbQueryToDatastoreQuery(
       dbQuery.select([]),
-      this.ds().createQuery(dbQuery.table),
-      this.getPropertyFilter(),
+      ds.createQuery(dbQuery.table),
+      await this.getPropertyFilter(),
     )
-    const aq = this.ds().createAggregationQuery(q).count('count')
+    const aq = ds.createAggregationQuery(q).count('count')
     const dsOpt = this.getRunQueryOptions(opt)
-    const [entities] = await this.ds().runAggregationQuery(aq, dsOpt)
+    const [entities] = await ds.runAggregationQuery(aq, dsOpt)
     return entities[0]?.count
   }
 
@@ -274,7 +279,8 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     q: Query,
     dsOpt: RunQueryOptions,
   ): Promise<RunQueryResult<ROW>> {
-    const [entities, queryResult] = await this.ds().runQuery(q, dsOpt)
+    const ds = await this.ds()
+    const [entities, queryResult] = await ds.runQuery(q, dsOpt)
 
     const rows = entities.map(e => this.mapId<ROW>(e))
 
@@ -284,37 +290,42 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  private runQueryStream<ROW extends ObjectWithId>(
-    q: Query,
+  override streamQuery<ROW extends ObjectWithId>(
+    dbQuery: DBQuery<ROW>,
     _opt?: DatastoreDBStreamOptions,
   ): ReadableTyped<ROW> {
-    const opt = {
-      ...this.cfg.streamOptions,
-      ..._opt,
-    }
-    const dsOpt = this.getRunQueryOptions(opt)
+    const transform = new Transform({
+      objectMode: true,
+      transform: (chunk, _, cb) => {
+        cb(null, this.mapId(chunk))
+      },
+    })
 
-    return (
-      opt.experimentalCursorStream
+    void this.ds().then(async ds => {
+      const q = dbQueryToDatastoreQuery(
+        dbQuery,
+        ds.createQuery(dbQuery.table),
+        await this.getPropertyFilter(),
+      )
+
+      const opt = {
+        ...this.cfg.streamOptions,
+        ..._opt,
+      }
+
+      ;(opt.experimentalCursorStream
         ? new DatastoreStreamReadable<ROW>(
             q,
             opt,
             commonLoggerMinLevel(this.cfg.logger, opt.debug ? 'log' : 'warn'),
           )
-        : (this.ds().runQueryStream(q, dsOpt) as ReadableTyped<ROW>)
-    ).map(chunk => this.mapId(chunk))
-  }
+        : (ds.runQueryStream(q, this.getRunQueryOptions(opt)) as ReadableTyped<ROW>)
+      )
+        .on('error', err => transform.emit('error', err))
+        .pipe(transform)
+    })
 
-  override streamQuery<ROW extends ObjectWithId>(
-    dbQuery: DBQuery<ROW>,
-    opt?: DatastoreDBStreamOptions,
-  ): ReadableTyped<ROW> {
-    const q = dbQueryToDatastoreQuery(
-      dbQuery,
-      this.ds().createQuery(dbQuery.table),
-      this.getPropertyFilter(),
-    )
-    return this.runQueryStream(q, opt)
+    return transform
   }
 
   // https://github.com/GoogleCloudPlatform/nodejs-getting-started/blob/master/2-structured-data/books/model-datastore.js
@@ -327,15 +338,16 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     rows: ROW[],
     opt: DatastoreDBSaveOptions<ROW> = {},
   ): Promise<void> {
+    const ds = await this.ds()
     const entities = rows.map(obj =>
-      this.toDatastoreEntity(table, obj, opt.excludeFromIndexes as string[]),
+      this.toDatastoreEntity(ds, table, obj, opt.excludeFromIndexes as string[]),
     )
 
     const method = methodMap[opt.saveMethod || 'upsert'] || 'save'
 
     const save = pRetryFn(
       async (batch: DatastorePayload<ROW>[]) => {
-        await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds())[method](batch)
+        await ((opt.tx as DatastoreDBTransaction)?.tx || ds)[method](batch)
       },
       this.getPRetryOptions(`DatastoreLib.saveBatch(${table})`),
     )
@@ -377,10 +389,11 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
       return await this.deleteByIds(q.table, ids, opt)
     }
 
+    const ds = await this.ds()
     const datastoreQuery = dbQueryToDatastoreQuery(
       q.select([]),
-      this.ds().createQuery(q.table),
-      this.getPropertyFilter(),
+      ds.createQuery(q.table),
+      await this.getPropertyFilter(),
     )
     const dsOpt = this.getRunQueryOptions(opt)
     const { rows } = await this.runDatastoreQuery<ROW>(datastoreQuery, dsOpt)
@@ -400,12 +413,13 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     ids: string[],
     opt: DatastoreDBOptions = {},
   ): Promise<number> {
-    const keys = ids.map(id => this.key(table, id))
+    const ds = await this.ds()
+    const keys = ids.map(id => this.key(ds, table, id))
 
     await pMap(
       _chunk(keys, MAX_ITEMS),
 
-      async batch => await ((opt.tx as DatastoreDBTransaction)?.tx || this.ds()).delete(batch),
+      async batch => await ((opt.tx as DatastoreDBTransaction)?.tx || ds).delete(batch),
       {
         concurrency: DATASTORE_RECOMMENDED_CONCURRENCY,
       },
@@ -417,8 +431,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     fn: DBTransactionFn,
     opt: CommonDBTransactionOptions = {},
   ): Promise<void> {
+    const ds = await this.ds()
     const { readOnly } = opt
-    const datastoreTx = this.ds().transaction({
+    const datastoreTx = ds.transaction({
       readOnly,
     })
 
@@ -434,8 +449,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   }
 
   async getAllStats(): Promise<DatastoreStats[]> {
-    const q = this.ds().createQuery('__Stat_Kind__')
-    const [statsArray] = await this.ds().runQuery(q)
+    const ds = await this.ds()
+    const q = ds.createQuery('__Stat_Kind__')
+    const [statsArray] = await ds.runQuery(q)
     return statsArray || []
   }
 
@@ -443,14 +459,15 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
    * Returns undefined e.g when Table is non-existing
    */
   async getStats(table: string): Promise<DatastoreStats | undefined> {
-    const { PropertyFilter } = this.getDatastoreLib()
+    const ds = await this.ds()
+    const propertyFilter = await this.getPropertyFilter()
 
-    const q = this.ds()
+    const q = ds
       .createQuery('__Stat_Kind__')
       // .filter('kind_name', table)
-      .filter(new PropertyFilter('kind_name', '=', table))
+      .filter(new propertyFilter('kind_name', '=', table))
       .limit(1)
-    const [statsArray] = await this.ds().runQuery(q)
+    const [statsArray] = await ds.runQuery(q)
     const [stats] = statsArray
     return stats
   }
@@ -461,11 +478,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
   }
 
   async getTableProperties(table: string): Promise<DatastorePropertyStats[]> {
-    const q = this.ds()
+    const ds = await this.ds()
+    const q = ds
       .createQuery('__Stat_PropertyType_PropertyName_Kind__')
       // .filter('kind_name', table)
-      .filter(new (this.getPropertyFilter())('kind_name', '=', table))
-    const [stats] = await this.ds().runQuery(q)
+      .filter(new (await this.getPropertyFilter())('kind_name', '=', table))
+    const [stats] = await ds.runQuery(q)
     return stats
   }
 
@@ -481,11 +499,12 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
 
   // if key field exists on entity, it will be used as key (prevent to duplication of numeric keyed entities)
   private toDatastoreEntity<T extends ObjectWithId>(
+    ds: Datastore,
     kind: string,
     o: T,
     excludeFromIndexes: string[] = [],
   ): DatastorePayload<T> {
-    const key = this.getDsKey(o) || this.key(kind, o.id)
+    const key = this.getDsKey(o) || this.key(ds, kind, o.id)
     const data = Object.assign({}, o) as any
     delete data.id
     delete data[this.KEY]
@@ -497,9 +516,9 @@ export class DatastoreDB extends BaseCommonDB implements CommonDB {
     }
   }
 
-  key(kind: string, id: string): Key {
+  key(ds: Datastore, kind: string, id: string): Key {
     _assert(id, `Cannot save "${kind}" entity without "id"`)
-    return this.ds().key([kind, id])
+    return ds.key([kind, id])
   }
 
   private getDsKey(o: any): Key | undefined {

package/src/datastoreKeyValueDB.ts

@@ -3,8 +3,8 @@ import { commonKeyValueDBFullSupport, DBQuery } from '@naturalcycles/db-lib'
 import type { ObjectWithId } from '@naturalcycles/js-lib'
 import { AppError } from '@naturalcycles/js-lib'
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib'
-import { DatastoreDB } from './datastore.db'
-import type { DatastoreDBCfg } from './datastore.model'
+import { DatastoreDB } from './datastore.db.js'
+import type { DatastoreDBCfg } from './datastore.model.js'
 
 interface KVObject {
   id: string

package/src/index.ts

@@ -1,3 +1,3 @@
-export * from './datastore.db'
-export * from './datastore.model'
-export * from './datastoreKeyValueDB'
+export * from './datastore.db.js'
+export * from './datastore.model.js'
+export * from './datastoreKeyValueDB.js'