@naturalcycles/mysql-lib 3.8.0

package/dist/schema/mysql.schema.util.d.ts

@@ -0,0 +1,24 @@
+import { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib';
+export interface MySQLTableStats {
+    Field: string;
+    Type: string;
+    Null: string;
+}
+export interface MySQLSchemaOptions {
+    /**
+     * @default 'InnoDB'
+     */
+    engine?: string;
+}
+/**
+ * It currently skips nullability and declares everything as "DEFAULT NULL".
+ */
+export declare function jsonSchemaToMySQLDDL(table: string, schema: JsonSchemaObject<any>, opt?: MySQLSchemaOptions): string;
+export declare function mysqlTableStatsToJsonSchemaField<T = any>(table: string, stats: MySQLTableStats[]): JsonSchemaRootObject<T>;
+/**
+ * Because MySQL doesn't support `.` in field names and escapes them as tableName + fieldName.
+ *
+ * @param name
+ */
+export declare function mapNameToMySQL(name: string): string;
+export declare function mapNameFromMySQL(name: string): string;

package/dist/schema/mysql.schema.util.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapNameFromMySQL = exports.mapNameToMySQL = exports.mysqlTableStatsToJsonSchemaField = exports.jsonSchemaToMySQLDDL = void 0;
+const mysql = require("mysql");
+/**
+ * It currently skips nullability and declares everything as "DEFAULT NULL".
+ */
+function jsonSchemaToMySQLDDL(table, schema, opt = {}) {
+    const { engine = 'InnoDB' } = opt;
+    const lines = [`CREATE TABLE ${mysql.escapeId(table)} (`];
+    const innerLines = Object.entries(schema.properties).map(([k, s]) => {
+        if (k === 'id') {
+            return `id VARCHAR(255) NOT NULL`;
+        }
+        let type;
+        if (s.type === 'string') {
+            // can specify isoDate later
+            type = 'LONGTEXT';
+        }
+        else if (s.type === 'integer') {
+            type = 'INT(11)';
+        }
+        else if (s.type === 'number') {
+            if (['unixTimestamp', 'int32'].includes(s.format)) {
+                type = 'INT(11)';
+            }
+            else {
+                type = 'FLOAT(11)';
+            }
+        }
+        else if (s.type === 'boolean') {
+            type = 'TINYINT(1)';
+        }
+        else if (s.instanceof === 'Buffer') {
+            type = 'LONGBLOB';
+        }
+        else if (s.type === 'null') {
+            type = 'VARCHAR(255)';
+        }
+        else if (s.type === 'array') {
+            type = 'LONGTEXT'; // to be JSON.stringified?
+        }
+        else if (s.type === 'object') {
+            type = 'LONGTEXT'; // to be JSON.stringified?
+        }
+        else {
+            // default
+            type = 'LONGTEXT';
+        }
+        const tokens = [mysql.escapeId(mapNameToMySQL(k)), type, `DEFAULT NULL`];
+        return tokens.join(' ');
+    });
+    innerLines.push(`PRIMARY KEY (id)`);
+    lines.push(innerLines.join(',\n'));
+    lines.push(`) ENGINE=${engine}`);
+    return lines.join('\n');
+}
+exports.jsonSchemaToMySQLDDL = jsonSchemaToMySQLDDL;
+function mysqlTableStatsToJsonSchemaField(table, stats) {
+    const s = {
+        $id: `${table}.schema.json`,
+        type: 'object',
+        properties: {},
+        required: [],
+        additionalProperties: true,
+    };
+    stats.forEach(stat => {
+        const name = stat.Field;
+        const t = stat.Type.toLowerCase();
+        const notNull = stat.Null?.toUpperCase() !== 'YES';
+        if (notNull) {
+            s.required.push(name);
+        }
+        if (t.includes('text') || t.includes('char')) {
+            s.properties[name] = { type: 'string' };
+        }
+        else if (t.includes('lob')) {
+            s.properties[name] = { instanceof: 'Buffer' };
+        }
+        else if (t.startsWith('tinyint') || t.includes('(1)')) {
+            s.properties[name] = { type: 'boolean' };
+        }
+        else if (t.startsWith('int(')) {
+            s.properties[name] = { type: 'integer' };
+        }
+        else if (t.startsWith('float')) {
+            s.properties[name] = { type: 'number' };
+        }
+        else {
+            console.log(s);
+            throw new Error(`Unknown mysql field type ${name} ${stat.Type}`);
+        }
+    });
+    return s;
+}
+exports.mysqlTableStatsToJsonSchemaField = mysqlTableStatsToJsonSchemaField;
+/**
+ * Because MySQL doesn't support `.` in field names and escapes them as tableName + fieldName.
+ *
+ * @param name
+ */
+function mapNameToMySQL(name) {
+    return name.replace(/\./g, '_dot_');
+}
+exports.mapNameToMySQL = mapNameToMySQL;
+function mapNameFromMySQL(name) {
+    return name.replace(/_dot_/g, '.');
+}
+exports.mapNameFromMySQL = mapNameFromMySQL;

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@naturalcycles/mysql-lib",
+  "scripts": {
+    "prepare": "husky install"
+  },
+  "peerDependencies": {
+    "@naturalcycles/db-lib": ">=7.0.0"
+  },
+  "dependencies": {
+    "@naturalcycles/db-lib": "^8.16.2",
+    "@naturalcycles/js-lib": "^14.47.0",
+    "@naturalcycles/nodejs-lib": "^12.37.0",
+    "@types/mysql": "^2.15.19",
+    "mysql": "^2.17.1"
+  },
+  "devDependencies": {
+    "@naturalcycles/dev-lib": "^12.7.5",
+    "@types/node": "^16.10.2",
+    "dotenv": "^10.0.0",
+    "jest": "^27.2.4"
+  },
+  "files": [
+    "dist",
+    "src",
+    "!src/test",
+    "!src/**/*.test.ts",
+    "!src/**/__snapshots__",
+    "!src/**/__exclude"
+  ],
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/NaturalCycles/mysql-lib"
+  },
+  "engines": {
+    "node": ">=14.15.0"
+  },
+  "version": "3.8.0",
+  "description": "MySQL client implementing CommonDB interface",
+  "keywords": [
+    "mysql",
+    "lib",
+    "db-lib",
+    "commondb"
+  ],
+  "author": "Natural Cycles Team",
+  "license": "MIT"
+}

package/readme.md

@@ -0,0 +1,14 @@
+## @naturalcycles/mysql-lib
+
+> MySQL client implementing CommonDB interface
+
+[![npm](https://img.shields.io/npm/v/@naturalcycles/mysql-lib/latest.svg)](https://www.npmjs.com/package/@naturalcycles/mysql-lib)
+[![](https://circleci.com/gh/NaturalCycles/mysql-lib.svg?style=shield&circle-token=123)](https://circleci.com/gh/NaturalCycles/mysql-lib)
+[![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
+
+# Packaging
+
+- `engines.node >= 10.13`: Latest Node.js LTS
+- `main: dist/index.js`: commonjs, es2018
+- `types: dist/index.d.ts`: typescript types
+- `/src` folder with source `*.ts` files included

package/src/dbAdapter.ts

@@ -0,0 +1,6 @@
+import { MysqlDB } from './mysql.db'
+
+export function getDBAdapter(cfgStr: string = '{}'): MysqlDB {
+  const cfg = JSON.parse(cfgStr)
+  return new MysqlDB(cfg)
+}

package/src/index.ts

@@ -0,0 +1,7 @@
+import { getDBAdapter } from './dbAdapter'
+import { MysqlDB, MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions } from './mysql.db'
+import { jsonSchemaToMySQLDDL } from './schema/mysql.schema.util'
+
+export type { MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions }
+
+export { MysqlDB, jsonSchemaToMySQLDDL, getDBAdapter }

package/src/mysql.db.ts

@@ -0,0 +1,312 @@
+import { Transform } from 'stream'
+import { promisify } from 'util'
+import {
+  BaseCommonDB,
+  BaseDBEntity,
+  CommonDB,
+  CommonDBCreateOptions,
+  CommonDBOptions,
+  CommonDBSaveOptions,
+  DBQuery,
+  ObjectWithId,
+  RunQueryResult,
+} from '@naturalcycles/db-lib'
+import {
+  _filterNullishValues,
+  _mapKeys,
+  _mapValues,
+  _Memo,
+  JsonSchemaObject,
+  JsonSchemaRootObject,
+} from '@naturalcycles/js-lib'
+import { Debug, ReadableTyped } from '@naturalcycles/nodejs-lib'
+import { white } from '@naturalcycles/nodejs-lib/dist/colors'
+import { Connection, Pool, PoolConfig, PoolConnection, QueryOptions, TypeCast } from 'mysql'
+import * as mysql from 'mysql'
+import { dbQueryToSQLDelete, dbQueryToSQLSelect, insertSQL } from './query.util'
+import {
+  jsonSchemaToMySQLDDL,
+  mapNameFromMySQL,
+  MySQLTableStats,
+  mysqlTableStatsToJsonSchemaField,
+} from './schema/mysql.schema.util'
+
+export interface MysqlDBOptions extends CommonDBOptions {}
+export interface MysqlDBSaveOptions extends CommonDBSaveOptions {}
+
+/**
+ * @default false / undefined
+ */
+export interface MysqlDBCfg extends PoolConfig {
+  /**
+   * If true - will log all produced SQL queries.
+   */
+  logSQL?: boolean
+
+  /**
+   * If true - will emit logs of connection events.
+   */
+  debugConnections?: boolean
+}
+
+const log = Debug('nc:mysql-lib')
+
+const BOOLEAN_TYPES = new Set(['TINY', 'TINYINT', 'INT'])
+const BOOLEAN_BIT_TYPES = new Set(['BIT'])
+
+const typeCast: TypeCast = (field, next) => {
+  // cast to boolean
+  if (field.length === 1) {
+    if (BOOLEAN_BIT_TYPES.has(field.type)) {
+      const b = field.buffer()
+      if (b === null || b === undefined) return undefined // null = undefined
+      return b[0] === 1 // 1 = true, 0 = false
+    }
+
+    if (BOOLEAN_TYPES.has(field.type)) {
+      const s = field.string()
+      // console.log(field.name, field.type, s, s?.charCodeAt(0))
+      if (s === null || s === undefined) return undefined // null = undefined
+      return s === '1' // 1 = true, 0 = false
+    }
+  }
+
+  return next()
+}
+
+export class MysqlDB extends BaseCommonDB implements CommonDB {
+  constructor(cfg: MysqlDBCfg = {}) {
+    super()
+    this.cfg = {
+      typeCast,
+      // charset: 'utf8mb4', // for emoji support
+      // host: 'localhost',
+      // user: MYSQL_USER,
+      // password: MYSQL_PW,
+      // database: MYSQL_DB,
+      ...cfg,
+    }
+  }
+
+  cfg!: MysqlDBCfg
+
+  override async ping(): Promise<void> {
+    const con = await this.getConnection()
+    con.release()
+  }
+
+  @_Memo()
+  pool(): Pool {
+    const pool = mysql.createPool(this.cfg)
+    const { host, database } = this.cfg
+    log(`connected to ${white(host + '/' + database)}`)
+
+    if (this.cfg.debugConnections) {
+      pool.on('acquire', con => {
+        log(`acquire(${con.threadId})`)
+      })
+
+      pool.on('connection', con => {
+        log(`connection(${con.threadId})`)
+      })
+
+      pool.on('enqueue', () => {
+        log(`enqueue`)
+      })
+
+      pool.on('release', con => {
+        log(`release(${con.threadId})`)
+      })
+    }
+
+    pool.on('error', err => {
+      log.error(`error`, err)
+    })
+
+    return pool
+  }
+
+  async close(): Promise<void> {
+    const pool = this.pool()
+    // eslint-disable-next-line @typescript-eslint/await-thenable
+    await promisify(pool.end.bind(pool))
+  }
+
+  /**
+   * Be careful to always call `con.release()` when you get connection with this method.
+   */
+  async getConnection(): Promise<PoolConnection> {
+    const pool = this.pool()
+    return await promisify(pool.getConnection.bind(pool))()
+  }
+
+  /**
+   * Manually create a single (not pool) connection.
+   * Be careful to manage this connection yourself (open, release, etc).
+   */
+  async createSingleConnection(): Promise<Connection> {
+    const con = mysql.createConnection(this.cfg)
+    await promisify(con.connect.bind(con))()
+    const { threadId } = con
+
+    if (this.cfg.debugConnections) {
+      log(`createSingleConnection(${threadId})`)
+
+      con.on('connect', () => log(`createSingleConnection(${threadId}).connect`))
+      con.on('drain', () => log(`createSingleConnection(${threadId}).drain`))
+      con.on('enqueue', () => log(`createSingleConnection(${threadId}).enqueue`))
+      con.on('end', () => log(`createSingleConnection(${threadId}).end`))
+    }
+
+    con.on('error', err => {
+      log.error(`createSingleConnection(${threadId}).error`, err)
+    })
+
+    return con
+  }
+
+  // GET
+  override async getByIds<ROW extends ObjectWithId>(
+    table: string,
+    ids: string[],
+    opt: MysqlDBOptions = {},
+  ): Promise<ROW[]> {
+    if (!ids.length) return []
+    const q = new DBQuery<ROW>(table).filterEq('id', ids)
+    const { rows } = await this.runQuery(q, opt)
+    return rows.map(r => _mapKeys(r, k => mapNameFromMySQL(k)) as any)
+  }
+
+  // QUERY
+  override async runQuery<ROW extends ObjectWithId, OUT = ROW>(
+    q: DBQuery<ROW>,
+    _opt: MysqlDBOptions = {},
+  ): Promise<RunQueryResult<OUT>> {
+    const sql = dbQueryToSQLSelect(q)
+    const rows = await this.runSQL<ROW[]>({ sql })
+    return {
+      rows: rows.map(r => _mapKeys(_filterNullishValues(r, true), k => mapNameFromMySQL(k)) as any),
+    }
+  }
+
+  async runSQL<RESULT>(q: QueryOptions): Promise<RESULT> {
+    if (this.cfg.logSQL) log(q.sql, q.values)
+
+    return await new Promise<RESULT>((resolve, reject) => {
+      this.pool().query(q, (err, res) => {
+        if (err) return reject(err)
+        resolve(res)
+      })
+    })
+  }
+
+  override async runQueryCount<ROW extends ObjectWithId>(
+    q: DBQuery<ROW>,
+    _opt?: CommonDBOptions,
+  ): Promise<number> {
+    const { rows } = await this.runQuery(q.select(['count(*) as _count']))
+    return (rows[0] as any)._count
+  }
+
+  override streamQuery<ROW extends ObjectWithId, OUT = ROW>(
+    q: DBQuery<ROW>,
+    _opt: MysqlDBOptions = {},
+  ): ReadableTyped<OUT> {
+    const sql = dbQueryToSQLSelect(q)
+
+    if (this.cfg.logSQL) log(`stream: ${sql}`)
+
+    // return this.streamSQL(sql, opt)
+    return this.pool()
+      .query(sql)
+      .stream()
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform(row, _encoding, cb) {
+            cb(null, _filterNullishValues(row, true))
+          },
+        }),
+      )
+  }
+
+  // SAVE
+  override async saveBatch<ROW extends BaseDBEntity>(
+    table: string,
+    _rows: ROW[],
+    _opt?: MysqlDBSaveOptions,
+  ): Promise<void> {
+    if (!_rows.length) return
+
+    // Stringify object values
+    const rows = _rows.map(row =>
+      _mapValues(row, v => {
+        return v && typeof v === 'object' && !Buffer.isBuffer(v) ? JSON.stringify(v) : v
+      }),
+    )
+
+    // inserts are split into multiple sentenses to respect the max_packet_size (1Mb usually)
+    const sqls = insertSQL(table, rows)
+
+    for await (const sql of sqls) {
+      await this.runSQL({ sql })
+    }
+  }
+
+  // DELETE
+  /**
+   * Limitation: always returns [], regardless of which rows are actually deleted
+   */
+  override async deleteByIds(table: string, ids: string[], _opt?: MysqlDBOptions): Promise<number> {
+    if (!ids.length) return 0
+    const sql = dbQueryToSQLDelete(new DBQuery(table).filterEq('id', ids))
+    const res = await this.runSQL<any>({ sql })
+    return res.affectedRows
+  }
+
+  override async deleteByQuery<ROW extends ObjectWithId>(
+    q: DBQuery<ROW>,
+    _opt?: CommonDBOptions,
+  ): Promise<number> {
+    const sql = dbQueryToSQLDelete(q)
+    const res = await this.runSQL<any>({ sql })
+    return res.affectedRows
+  }
+
+  /**
+   * Use with caution!
+   */
+  async dropTable(table: string): Promise<void> {
+    await this.runSQL({ sql: `DROP TABLE IF EXISTS ${table}` })
+  }
+
+  /**
+   * dropIfExists=true needed as a safety check
+   */
+  override async createTable(
+    table: string,
+    schema: JsonSchemaObject,
+    opt: CommonDBCreateOptions = {},
+  ): Promise<void> {
+    if (opt.dropIfExists) await this.dropTable(table)
+
+    const sql = jsonSchemaToMySQLDDL(table, schema)
+    await this.runSQL({ sql })
+  }
+
+  override async getTables(): Promise<string[]> {
+    return (await this.runSQL<Record<any, any>[]>({ sql: `show tables` }))
+      .map(r => Object.values(r)[0])
+      .filter(Boolean)
+  }
+
+  override async getTableSchema<ROW extends ObjectWithId>(
+    table: string,
+  ): Promise<JsonSchemaRootObject<ROW>> {
+    const stats = await this.runSQL<MySQLTableStats[]>({
+      sql: `describe ${mysql.escapeId(table)}`,
+    })
+
+    return mysqlTableStatsToJsonSchemaField<ROW>(table, stats)
+  }
+}