@naturalcycles/mysql-lib 3.9.2 → 3.10.0

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 import { getDBAdapter } from './dbAdapter';
 import { MysqlDB, MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions } from './mysql.db';
+import { MySQLKeyValueDB } from './mysqlKeyValueDB';
 import { jsonSchemaToMySQLDDL } from './schema/mysql.schema.util';
 export type { MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions };
-export { MysqlDB, jsonSchemaToMySQLDDL, getDBAdapter };
+export { MysqlDB, MySQLKeyValueDB, jsonSchemaToMySQLDDL, getDBAdapter };

package/dist/index.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDBAdapter = exports.jsonSchemaToMySQLDDL = exports.MysqlDB = void 0;
+exports.getDBAdapter = exports.jsonSchemaToMySQLDDL = exports.MySQLKeyValueDB = exports.MysqlDB = void 0;
 const dbAdapter_1 = require("./dbAdapter");
 Object.defineProperty(exports, "getDBAdapter", { enumerable: true, get: function () { return dbAdapter_1.getDBAdapter; } });
 const mysql_db_1 = require("./mysql.db");
 Object.defineProperty(exports, "MysqlDB", { enumerable: true, get: function () { return mysql_db_1.MysqlDB; } });
+const mysqlKeyValueDB_1 = require("./mysqlKeyValueDB");
+Object.defineProperty(exports, "MySQLKeyValueDB", { enumerable: true, get: function () { return mysqlKeyValueDB_1.MySQLKeyValueDB; } });
 const mysql_schema_util_1 = require("./schema/mysql.schema.util");
 Object.defineProperty(exports, "jsonSchemaToMySQLDDL", { enumerable: true, get: function () { return mysql_schema_util_1.jsonSchemaToMySQLDDL; } });

package/dist/mysql.db.d.ts

@@ -1,7 +1,7 @@
 import { BaseCommonDB, CommonDB, CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, DBQuery, RunQueryResult } from '@naturalcycles/db-lib';
 import { AnyObjectWithId, CommonLogger, JsonSchemaObject, JsonSchemaRootObject, ObjectWithId } from '@naturalcycles/js-lib';
 import { ReadableTyped } from '@naturalcycles/nodejs-lib';
-import { Connection, Pool, PoolConfig, PoolConnection, QueryOptions } from 'mysql';
+import { Connection, Pool, PoolConfig, PoolConnection, QueryOptions, TypeCast } from 'mysql';
 export interface MysqlDBOptions extends CommonDBOptions {
 }
 export interface MysqlDBSaveOptions<ROW extends ObjectWithId = AnyObjectWithId> extends CommonDBSaveOptions<ROW> {
@@ -23,6 +23,7 @@ export interface MysqlDBCfg extends PoolConfig {
      */
     logger?: CommonLogger;
 }
+export declare const typeCast: TypeCast;
 export declare class MysqlDB extends BaseCommonDB implements CommonDB {
     constructor(cfg?: MysqlDBCfg);
     cfg: MysqlDBCfg & {

package/dist/mysql.db.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MysqlDB = void 0;
+exports.MysqlDB = exports.typeCast = void 0;
 const tslib_1 = require("tslib");
 const stream_1 = require("stream");
 const util_1 = require("util");
@@ -31,11 +31,12 @@ const typeCast = (field, next) => {
     }
     return next();
 };
+exports.typeCast = typeCast;
 class MysqlDB extends db_lib_1.BaseCommonDB {
     constructor(cfg = {}) {
         super();
         this.cfg = {
-            typeCast,
+            typeCast: exports.typeCast,
             // charset: 'utf8mb4', // for emoji support
             // host: 'localhost',
             // user: MYSQL_USER,

package/dist/mysqlKeyValueDB.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { MysqlDB, MysqlDBCfg } from './mysql.db';
+export declare class MySQLKeyValueDB implements CommonKeyValueDB {
+    constructor(cfg?: MysqlDBCfg);
+    cfg: MysqlDBCfg;
+    db: MysqlDB;
+    ping(): Promise<void>;
+    close(): Promise<void>;
+    createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
+    getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
+    /**
+     * Use with caution!
+     */
+    dropTable(table: string): Promise<void>;
+    deleteByIds(table: string, ids: string[]): Promise<void>;
+    saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void>;
+    streamIds(table: string, limit?: number): ReadableTyped<string>;
+    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
+    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    beginTransaction(): Promise<void>;
+    endTransaction(): Promise<void>;
+    count(table: string): Promise<number>;
+}

package/dist/mysqlKeyValueDB.js

@@ -0,0 +1,128 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MySQLKeyValueDB = void 0;
+const stream_1 = require("stream");
+const js_lib_1 = require("@naturalcycles/js-lib");
+const mysql_db_1 = require("./mysql.db");
+class MySQLKeyValueDB {
+    constructor(cfg = {}) {
+        this.cfg = {
+            typeCast: mysql_db_1.typeCast,
+            ...cfg,
+        };
+        this.db = new mysql_db_1.MysqlDB(this.cfg);
+    }
+    async ping() {
+        await this.db.ping();
+    }
+    async close() {
+        await this.db.close();
+    }
+    async createTable(table, opt = {}) {
+        if (opt.dropIfExists)
+            await this.dropTable(table);
+        // On blob sizes: https://tableplus.com/blog/2019/10/tinyblob-blob-mediumblob-longblob.html
+        // LONGBLOB supports up to 4gb
+        // MEDIUMBLOB up to 16Mb
+        const sql = `create table ${table} (id VARCHAR(64) PRIMARY KEY, v LONGBLOB NOT NULL)`;
+        this.db.cfg.logger.log(sql);
+        await this.db.runSQL({ sql });
+    }
+    async getByIds(table, ids) {
+        if (!ids.length)
+            return [];
+        const sql = `SELECT id,v FROM ${table} where id in (${ids.map(id => `"${id}"`).join(',')})`;
+        const rows = await this.db.runSQL({ sql });
+        return rows.map(({ id, v }) => [id, v]);
+    }
+    /**
+     * Use with caution!
+     */
+    async dropTable(table) {
+        await this.db.runSQL({ sql: `DROP TABLE IF EXISTS ${table}` });
+    }
+    async deleteByIds(table, ids) {
+        const sql = `DELETE FROM ${table} WHERE id in (${ids.map(id => `"${id}"`).join(',')})`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(sql);
+        await this.db.runSQL({ sql });
+    }
+    async saveBatch(table, entries) {
+        const statements = entries.map(([id, buf]) => {
+            return {
+                sql: `INSERT INTO ${table} (id, v) VALUES (?, ?)`,
+                values: [id, buf],
+            };
+        });
+        await (0, js_lib_1.pMap)(statements, async (statement) => {
+            if (this.cfg.debug)
+                this.db.cfg.logger.log(statement.sql);
+            await this.db.runSQL(statement);
+        });
+    }
+    streamIds(table, limit) {
+        let sql = `SELECT id FROM ${table}`;
+        if (limit)
+            sql += ` LIMIT ${limit}`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(`stream: ${sql}`);
+        return this.db
+            .pool()
+            .query(sql)
+            .stream()
+            .pipe(new stream_1.Transform({
+            objectMode: true,
+            transform(row, _, cb) {
+                cb(null, row.id);
+            },
+        }));
+    }
+    streamValues(table, limit) {
+        let sql = `SELECT v FROM ${table}`;
+        if (limit)
+            sql += ` LIMIT ${limit}`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(`stream: ${sql}`);
+        return this.db
+            .pool()
+            .query(sql)
+            .stream()
+            .pipe(new stream_1.Transform({
+            objectMode: true,
+            transform(row, _, cb) {
+                cb(null, row.v);
+            },
+        }));
+    }
+    streamEntries(table, limit) {
+        let sql = `SELECT id,v FROM ${table}`;
+        if (limit)
+            sql += ` LIMIT ${limit}`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(`stream: ${sql}`);
+        return this.db
+            .pool()
+            .query(sql)
+            .stream()
+            .pipe(new stream_1.Transform({
+            objectMode: true,
+            transform(row, _, cb) {
+                cb(null, [row.id, row.v]);
+            },
+        }));
+    }
+    async beginTransaction() {
+        await this.db.runSQL({ sql: `BEGIN TRANSACTION` });
+    }
+    async endTransaction() {
+        await this.db.runSQL({ sql: `END TRANSACTION` });
+    }
+    async count(table) {
+        const sql = `SELECT count(*) as cnt FROM ${table}`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(sql);
+        const rows = await this.db.runSQL({ sql });
+        return rows[0].cnt;
+    }
+}
+exports.MySQLKeyValueDB = MySQLKeyValueDB;

package/package.json

@@ -36,7 +36,7 @@
   "engines": {
     "node": ">=14.15.0"
   },
-  "version": "3.9.2",
+  "version": "3.10.0",
   "description": "MySQL client implementing CommonDB interface",
   "keywords": [
     "mysql",

package/src/index.ts

@@ -1,7 +1,8 @@
 import { getDBAdapter } from './dbAdapter'
 import { MysqlDB, MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions } from './mysql.db'
+import { MySQLKeyValueDB } from './mysqlKeyValueDB'
 import { jsonSchemaToMySQLDDL } from './schema/mysql.schema.util'
 
 export type { MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions }
 
-export { MysqlDB, jsonSchemaToMySQLDDL, getDBAdapter }
+export { MysqlDB, MySQLKeyValueDB, jsonSchemaToMySQLDDL, getDBAdapter }

package/src/mysql.db.ts

@@ -60,7 +60,7 @@ export interface MysqlDBCfg extends PoolConfig {
 const BOOLEAN_TYPES = new Set(['TINY', 'TINYINT', 'INT'])
 const BOOLEAN_BIT_TYPES = new Set(['BIT'])
 
-const typeCast: TypeCast = (field, next) => {
+export const typeCast: TypeCast = (field, next) => {
   // cast to boolean
   if (field.length === 1) {
     if (BOOLEAN_BIT_TYPES.has(field.type)) {

package/src/mysqlKeyValueDB.ts

@@ -0,0 +1,155 @@
+import { Transform } from 'stream'
+import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib'
+import { pMap } from '@naturalcycles/js-lib'
+import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import { QueryOptions } from 'mysql'
+import { MysqlDB, MysqlDBCfg, typeCast } from './mysql.db'
+
+interface KeyValueObject {
+  id: string
+  v: Buffer
+}
+
+export class MySQLKeyValueDB implements CommonKeyValueDB {
+  constructor(cfg: MysqlDBCfg = {}) {
+    this.cfg = {
+      typeCast,
+      ...cfg,
+    }
+
+    this.db = new MysqlDB(this.cfg)
+  }
+
+  cfg: MysqlDBCfg
+
+  db: MysqlDB
+
+  async ping(): Promise<void> {
+    await this.db.ping()
+  }
+
+  async close(): Promise<void> {
+    await this.db.close()
+  }
+
+  async createTable(table: string, opt: CommonDBCreateOptions = {}): Promise<void> {
+    if (opt.dropIfExists) await this.dropTable(table)
+
+    // On blob sizes: https://tableplus.com/blog/2019/10/tinyblob-blob-mediumblob-longblob.html
+    // LONGBLOB supports up to 4gb
+    // MEDIUMBLOB up to 16Mb
+    const sql = `create table ${table} (id VARCHAR(64) PRIMARY KEY, v LONGBLOB NOT NULL)`
+    this.db.cfg.logger.log(sql)
+    await this.db.runSQL({ sql })
+  }
+
+  async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
+    if (!ids.length) return []
+
+    const sql = `SELECT id,v FROM ${table} where id in (${ids.map(id => `"${id}"`).join(',')})`
+
+    const rows = await this.db.runSQL<KeyValueObject[]>({ sql })
+
+    return rows.map(({ id, v }) => [id, v])
+  }
+
+  /**
+   * Use with caution!
+   */
+  async dropTable(table: string): Promise<void> {
+    await this.db.runSQL({ sql: `DROP TABLE IF EXISTS ${table}` })
+  }
+
+  async deleteByIds(table: string, ids: string[]): Promise<void> {
+    const sql = `DELETE FROM ${table} WHERE id in (${ids.map(id => `"${id}"`).join(',')})`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(sql)
+    await this.db.runSQL({ sql })
+  }
+
+  async saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void> {
+    const statements: QueryOptions[] = entries.map(([id, buf]) => {
+      return {
+        sql: `INSERT INTO ${table} (id, v) VALUES (?, ?)`,
+        values: [id, buf],
+      }
+    })
+
+    await pMap(statements, async statement => {
+      if (this.cfg.debug) this.db.cfg.logger.log(statement.sql)
+      await this.db.runSQL(statement)
+    })
+  }
+
+  streamIds(table: string, limit?: number): ReadableTyped<string> {
+    let sql = `SELECT id FROM ${table}`
+    if (limit) sql += ` LIMIT ${limit}`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(`stream: ${sql}`)
+
+    return this.db
+      .pool()
+      .query(sql)
+      .stream()
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform(row, _, cb) {
+            cb(null, row.id)
+          },
+        }),
+      )
+  }
+
+  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
+    let sql = `SELECT v FROM ${table}`
+    if (limit) sql += ` LIMIT ${limit}`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(`stream: ${sql}`)
+
+    return this.db
+      .pool()
+      .query(sql)
+      .stream()
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform(row, _, cb) {
+            cb(null, row.v)
+          },
+        }),
+      )
+  }
+
+  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
+    let sql = `SELECT id,v FROM ${table}`
+    if (limit) sql += ` LIMIT ${limit}`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(`stream: ${sql}`)
+
+    return this.db
+      .pool()
+      .query(sql)
+      .stream()
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform(row, _, cb) {
+            cb(null, [row.id, row.v])
+          },
+        }),
+      )
+  }
+
+  async beginTransaction(): Promise<void> {
+    await this.db.runSQL({ sql: `BEGIN TRANSACTION` })
+  }
+
+  async endTransaction(): Promise<void> {
+    await this.db.runSQL({ sql: `END TRANSACTION` })
+  }
+
+  async count(table: string): Promise<number> {
+    const sql = `SELECT count(*) as cnt FROM ${table}`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(sql)
+
+    const rows = await this.db.runSQL<{ cnt: number }[]>({ sql })
+    return rows[0]!.cnt
+  }
+}