@naturalcycles/mysql-lib 3.9.0 → 3.11.0

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 import { getDBAdapter } from './dbAdapter';
 import { MysqlDB, MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions } from './mysql.db';
+import { MySQLKeyValueDB } from './mysqlKeyValueDB';
 import { jsonSchemaToMySQLDDL } from './schema/mysql.schema.util';
 export type { MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions };
-export { MysqlDB, jsonSchemaToMySQLDDL, getDBAdapter };
+export { MysqlDB, MySQLKeyValueDB, jsonSchemaToMySQLDDL, getDBAdapter };

package/dist/index.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getDBAdapter = exports.jsonSchemaToMySQLDDL = exports.MysqlDB = void 0;
+exports.getDBAdapter = exports.jsonSchemaToMySQLDDL = exports.MySQLKeyValueDB = exports.MysqlDB = void 0;
 const dbAdapter_1 = require("./dbAdapter");
 Object.defineProperty(exports, "getDBAdapter", { enumerable: true, get: function () { return dbAdapter_1.getDBAdapter; } });
 const mysql_db_1 = require("./mysql.db");
 Object.defineProperty(exports, "MysqlDB", { enumerable: true, get: function () { return mysql_db_1.MysqlDB; } });
+const mysqlKeyValueDB_1 = require("./mysqlKeyValueDB");
+Object.defineProperty(exports, "MySQLKeyValueDB", { enumerable: true, get: function () { return mysqlKeyValueDB_1.MySQLKeyValueDB; } });
 const mysql_schema_util_1 = require("./schema/mysql.schema.util");
 Object.defineProperty(exports, "jsonSchemaToMySQLDDL", { enumerable: true, get: function () { return mysql_schema_util_1.jsonSchemaToMySQLDDL; } });

package/dist/mysql.db.d.ts

@@ -43,6 +43,11 @@ export declare class MysqlDB extends BaseCommonDB implements CommonDB {
     getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: MysqlDBOptions): Promise<ROW[]>;
     runQuery<ROW extends ObjectWithId, OUT = ROW>(q: DBQuery<ROW>, _opt?: MysqlDBOptions): Promise<RunQueryResult<OUT>>;
     runSQL<RESULT>(q: QueryOptions): Promise<RESULT>;
+    /**
+     * Allows to run semicolon-separated "SQL file".
+     * E.g "ddl reset script".
+     */
+    runSQLString(s: string): Promise<void>;
     runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
     streamQuery<ROW extends ObjectWithId, OUT = ROW>(q: DBQuery<ROW>, _opt?: MysqlDBOptions): ReadableTyped<OUT>;
     saveBatch<ROW extends ObjectWithId>(table: string, _rows: ROW[], _opt?: MysqlDBSaveOptions<ROW>): Promise<void>;

package/dist/mysql.db.js

@@ -75,7 +75,7 @@ class MysqlDB extends db_lib_1.BaseCommonDB {
     async close() {
         const pool = this.pool();
         // eslint-disable-next-line @typescript-eslint/await-thenable
-        await (0, util_1.promisify)(pool.end.bind(pool));
+        await (0, util_1.promisify)(pool.end.bind(pool))();
         this.cfg.logger.log('closed');
     }
     /**
@@ -116,14 +116,20 @@ class MysqlDB extends db_lib_1.BaseCommonDB {
     // QUERY
     async runQuery(q, _opt = {}) {
         const sql = (0, query_util_1.dbQueryToSQLSelect)(q);
-        const rows = await this.runSQL({ sql });
+        const rows = (await this.runSQL({ sql })).map(row => (0, js_lib_1._mapKeys)((0, js_lib_1._filterUndefinedValues)(row, true), k => (0, mysql_schema_util_1.mapNameFromMySQL)(k)));
+        // edge case where 0 fields are selected
+        if (q._selectedFieldNames?.length === 0) {
+            return {
+                rows: rows.map(_ => ({})),
+            };
+        }
         return {
-            rows: rows.map(r => (0, js_lib_1._mapKeys)((0, js_lib_1._filterNullishValues)(r, true), k => (0, mysql_schema_util_1.mapNameFromMySQL)(k))),
+            rows,
         };
     }
     async runSQL(q) {
         if (this.cfg.logSQL)
-            this.cfg.logger.log(q.sql, q.values);
+            this.cfg.logger.log(...[q.sql, q.values].filter(Boolean));
         return await new Promise((resolve, reject) => {
             this.pool().query(q, (err, res) => {
                 if (err)
@@ -132,6 +138,19 @@ class MysqlDB extends db_lib_1.BaseCommonDB {
             });
         });
     }
+    /**
+     * Allows to run semicolon-separated "SQL file".
+     * E.g "ddl reset script".
+     */
+    async runSQLString(s) {
+        const queries = s
+            .split(';')
+            .map(s => s.trim())
+            .filter(Boolean);
+        for await (const sql of queries) {
+            await this.runSQL({ sql });
+        }
+    }
     async runQueryCount(q, _opt) {
         const { rows } = await this.runQuery(q.select(['count(*) as _count']));
         return rows[0]._count;
@@ -147,7 +166,7 @@ class MysqlDB extends db_lib_1.BaseCommonDB {
             .pipe(new stream_1.Transform({
             objectMode: true,
             transform(row, _encoding, cb) {
-                cb(null, (0, js_lib_1._filterNullishValues)(row, true));
+                cb(null, (0, js_lib_1._filterUndefinedValues)(row, true));
             },
         }));
     }
@@ -156,11 +175,11 @@ class MysqlDB extends db_lib_1.BaseCommonDB {
         if (!_rows.length)
             return;
         // Stringify object values
-        const rows = _rows.map(row => (0, js_lib_1._mapValues)(row, v => {
+        const rows = _rows.map(row => (0, js_lib_1._mapValues)(row, (_k, v) => {
             return v && typeof v === 'object' && !Buffer.isBuffer(v) ? JSON.stringify(v) : v;
         }));
         // inserts are split into multiple sentenses to respect the max_packet_size (1Mb usually)
-        const sqls = (0, query_util_1.insertSQL)(table, rows, 'INSERT', this.cfg.logger);
+        const sqls = (0, query_util_1.insertSQL)(table, rows, 'REPLACE', this.cfg.logger);
         for await (const sql of sqls) {
             await this.runSQL({ sql });
         }

package/dist/mysqlKeyValueDB.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { MysqlDB, MysqlDBCfg } from './mysql.db';
+export declare class MySQLKeyValueDB implements CommonKeyValueDB {
+    cfg: MysqlDBCfg;
+    constructor(cfg: MysqlDBCfg);
+    db: MysqlDB;
+    ping(): Promise<void>;
+    close(): Promise<void>;
+    createTable(table: string, opt?: CommonDBCreateOptions): Promise<void>;
+    getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]>;
+    /**
+     * Use with caution!
+     */
+    dropTable(table: string): Promise<void>;
+    deleteByIds(table: string, ids: string[]): Promise<void>;
+    saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void>;
+    streamIds(table: string, limit?: number): ReadableTyped<string>;
+    streamValues(table: string, limit?: number): ReadableTyped<Buffer>;
+    streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple>;
+    beginTransaction(): Promise<void>;
+    endTransaction(): Promise<void>;
+    count(table: string): Promise<number>;
+}

package/dist/mysqlKeyValueDB.js

@@ -0,0 +1,125 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MySQLKeyValueDB = void 0;
+const stream_1 = require("stream");
+const js_lib_1 = require("@naturalcycles/js-lib");
+const mysql_db_1 = require("./mysql.db");
+class MySQLKeyValueDB {
+    constructor(cfg) {
+        this.cfg = cfg;
+        this.db = new mysql_db_1.MysqlDB(this.cfg);
+    }
+    async ping() {
+        await this.db.ping();
+    }
+    async close() {
+        await this.db.close();
+    }
+    async createTable(table, opt = {}) {
+        if (opt.dropIfExists)
+            await this.dropTable(table);
+        // On blob sizes: https://tableplus.com/blog/2019/10/tinyblob-blob-mediumblob-longblob.html
+        // LONGBLOB supports up to 4gb
+        // MEDIUMBLOB up to 16Mb
+        const sql = `create table ${table} (id VARCHAR(64) PRIMARY KEY, v LONGBLOB NOT NULL)`;
+        this.db.cfg.logger.log(sql);
+        await this.db.runSQL({ sql });
+    }
+    async getByIds(table, ids) {
+        if (!ids.length)
+            return [];
+        const sql = `SELECT id,v FROM ${table} where id in (${ids.map(id => `"${id}"`).join(',')})`;
+        const rows = await this.db.runSQL({ sql });
+        return rows.map(({ id, v }) => [id, v]);
+    }
+    /**
+     * Use with caution!
+     */
+    async dropTable(table) {
+        await this.db.runSQL({ sql: `DROP TABLE IF EXISTS ${table}` });
+    }
+    async deleteByIds(table, ids) {
+        const sql = `DELETE FROM ${table} WHERE id in (${ids.map(id => `"${id}"`).join(',')})`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(sql);
+        await this.db.runSQL({ sql });
+    }
+    async saveBatch(table, entries) {
+        const statements = entries.map(([id, buf]) => {
+            return {
+                sql: `INSERT INTO ${table} (id, v) VALUES (?, ?)`,
+                values: [id, buf],
+            };
+        });
+        await (0, js_lib_1.pMap)(statements, async (statement) => {
+            if (this.cfg.debug)
+                this.db.cfg.logger.log(statement.sql);
+            await this.db.runSQL(statement);
+        });
+    }
+    streamIds(table, limit) {
+        let sql = `SELECT id FROM ${table}`;
+        if (limit)
+            sql += ` LIMIT ${limit}`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(`stream: ${sql}`);
+        return this.db
+            .pool()
+            .query(sql)
+            .stream()
+            .pipe(new stream_1.Transform({
+            objectMode: true,
+            transform(row, _, cb) {
+                cb(null, row.id);
+            },
+        }));
+    }
+    streamValues(table, limit) {
+        let sql = `SELECT v FROM ${table}`;
+        if (limit)
+            sql += ` LIMIT ${limit}`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(`stream: ${sql}`);
+        return this.db
+            .pool()
+            .query(sql)
+            .stream()
+            .pipe(new stream_1.Transform({
+            objectMode: true,
+            transform(row, _, cb) {
+                cb(null, row.v);
+            },
+        }));
+    }
+    streamEntries(table, limit) {
+        let sql = `SELECT id,v FROM ${table}`;
+        if (limit)
+            sql += ` LIMIT ${limit}`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(`stream: ${sql}`);
+        return this.db
+            .pool()
+            .query(sql)
+            .stream()
+            .pipe(new stream_1.Transform({
+            objectMode: true,
+            transform(row, _, cb) {
+                cb(null, [row.id, row.v]);
+            },
+        }));
+    }
+    async beginTransaction() {
+        await this.db.runSQL({ sql: `BEGIN TRANSACTION` });
+    }
+    async endTransaction() {
+        await this.db.runSQL({ sql: `END TRANSACTION` });
+    }
+    async count(table) {
+        const sql = `SELECT count(*) as cnt FROM ${table}`;
+        if (this.cfg.logSQL)
+            this.db.cfg.logger.log(sql);
+        const rows = await this.db.runSQL({ sql });
+        return rows[0].cnt;
+    }
+}
+exports.MySQLKeyValueDB = MySQLKeyValueDB;

package/dist/query.util.d.ts

@@ -6,7 +6,7 @@ export declare function dbQueryToSQLDelete(q: DBQuery<any>): string;
 /**
  * Returns array of sql statements to respect the max sql size.
  */
-export declare function insertSQL(table: string, records: Record<any, any>[], verb?: 'INSERT' | 'REPLACE', logger?: CommonLogger): string[];
+export declare function insertSQL(table: string, rows: Record<any, any>[], verb?: 'INSERT' | 'REPLACE', logger?: CommonLogger): string[];
 export declare function insertSQLSingle(table: string, record: Record<any, any>): string;
 export declare function insertSQLSetSingle(table: string, record: Record<any, any>): QueryOptions;
 export declare function dbQueryToSQLUpdate(q: DBQuery<any>, record: Record<any, any>): string;

package/dist/query.util.js

@@ -12,7 +12,7 @@ function dbQueryToSQLSelect(q) {
     // filters
     tokens.push(...whereTokens(q));
     // order
-    tokens.push(...orderTokens(q));
+    tokens.push(...groupOrderTokens(q));
     // offset/limit
     tokens.push(...offsetLimitTokens(q));
     return tokens.join(' ');
@@ -30,12 +30,12 @@ exports.dbQueryToSQLDelete = dbQueryToSQLDelete;
 /**
  * Returns array of sql statements to respect the max sql size.
  */
-function insertSQL(table, records, verb = 'INSERT', logger = console) {
+function insertSQL(table, rows, verb = 'INSERT', logger = console) {
     // INSERT INTO table_name (column1, column2, column3, ...)
     // VALUES (value1, value2, value3, ...);
     // eslint-disable-next-line unicorn/no-array-reduce
-    const fieldSet = records.reduce((set, rec) => {
-        Object.keys(rec).forEach(field => set.add(field));
+    const fieldSet = rows.reduce((set, row) => {
+        Object.keys(row).forEach(field => set.add(field));
         return set;
     }, new Set());
     const fields = [...fieldSet];
@@ -46,7 +46,7 @@ function insertSQL(table, records, verb = 'INSERT', logger = console) {
         `(` + [...fields].map(f => mysql.escapeId((0, mysql_schema_util_1.mapNameToMySQL)(f))).join(',') + `)`,
         `VALUES\n`,
     ].join(' ');
-    const valueRows = records.map(rec => {
+    const valueRows = rows.map(rec => {
         return `(` + fields.map(k => mysql.escape(rec[k])).join(',') + `)`;
     });
     const full = start + valueRows.join(',\n');
@@ -124,7 +124,13 @@ function selectTokens(q) {
         fields = q._selectedFieldNames.length ? q._selectedFieldNames : ['id'];
     }
     // We don't do `escapeId` cause it'll ruin e.g SELECT `count *` FROM ...
-    return [`SELECT`, fields.map(f => (0, mysql_schema_util_1.mapNameToMySQL)(f)).join(', '), `FROM`, mysql.escapeId(q.table)];
+    return [
+        `SELECT`,
+        q._distinct && 'DISTINCT',
+        fields.map(f => (0, mysql_schema_util_1.mapNameToMySQL)(f)).join(', '),
+        `FROM`,
+        mysql.escapeId(q.table),
+    ].filter(Boolean);
 }
 function offsetLimitTokens(q) {
     const tokens = [];
@@ -137,13 +143,15 @@ function offsetLimitTokens(q) {
     }
     return tokens;
 }
-function orderTokens(q) {
-    if (!q._orders.length)
-        return [];
-    return [
-        `ORDER BY`,
-        q._orders.map(o => `\`${o.name}\` ${o.descending ? 'DESC' : 'ASC'}`).join(', '),
-    ];
+function groupOrderTokens(q) {
+    const t = [];
+    if (q._groupByFieldNames?.length) {
+        t.push(`GROUP BY`, q._groupByFieldNames.map(c => `\`${c}\``).join(', '));
+    }
+    if (q._orders.length) {
+        t.push(`ORDER BY`, q._orders.map(o => `\`${o.name}\` ${o.descending ? 'DESC' : 'ASC'}`).join(', '));
+    }
+    return t;
 }
 const OP_MAP = {
     '==': '=',

package/dist/schema/mysql.schema.util.js

@@ -80,7 +80,7 @@ function mysqlTableStatsToJsonSchemaField(table, stats, logger) {
         else if (t.startsWith('tinyint') || t.includes('(1)')) {
             s.properties[name] = { type: 'boolean' };
         }
-        else if (t.startsWith('int(')) {
+        else if (t === 'int' || t.startsWith('int(')) {
             s.properties[name] = { type: 'integer' };
         }
         else if (t.startsWith('float')) {

package/package.json

@@ -12,8 +12,8 @@
   },
   "devDependencies": {
     "@naturalcycles/dev-lib": "^12.7.5",
-    "@types/node": "^16.10.2",
-    "dotenv": "^10.0.0",
+    "@types/node": "^17.0.5",
+    "dotenv": "^14.1.0",
     "jest": "^27.2.4"
   },
   "files": [
@@ -36,7 +36,7 @@
   "engines": {
     "node": ">=14.15.0"
   },
-  "version": "3.9.0",
+  "version": "3.11.0",
   "description": "MySQL client implementing CommonDB interface",
   "keywords": [
     "mysql",

package/readme.md

@@ -3,12 +3,33 @@
 > MySQL client implementing CommonDB interface
 
 [![npm](https://img.shields.io/npm/v/@naturalcycles/mysql-lib/latest.svg)](https://www.npmjs.com/package/@naturalcycles/mysql-lib)
-[![](https://circleci.com/gh/NaturalCycles/mysql-lib.svg?style=shield&circle-token=123)](https://circleci.com/gh/NaturalCycles/mysql-lib)
 [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier)
 
-# Packaging
+# Quick start
 
-- `engines.node >= 10.13`: Latest Node.js LTS
-- `main: dist/index.js`: commonjs, es2018
-- `types: dist/index.d.ts`: typescript types
-- `/src` folder with source `*.ts` files included
+todo
+
+# Develop
+
+Install MySQL server+client on Mac:
+
+```shell
+
+brew install mysql
+
+# start
+
+mysql.server start
+
+# secure
+
+mysql_secure_installation
+
+# connect
+
+mysql -uroot
+
+# Default options are read from the following files in the given order:
+# /etc/my.cnf /etc/mysql/my.cnf /opt/homebrew/etc/my.cnf ~/.my.cnf
+
+```

package/src/index.ts

@@ -1,7 +1,8 @@
 import { getDBAdapter } from './dbAdapter'
 import { MysqlDB, MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions } from './mysql.db'
+import { MySQLKeyValueDB } from './mysqlKeyValueDB'
 import { jsonSchemaToMySQLDDL } from './schema/mysql.schema.util'
 
 export type { MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions }
 
-export { MysqlDB, jsonSchemaToMySQLDDL, getDBAdapter }
+export { MysqlDB, MySQLKeyValueDB, jsonSchemaToMySQLDDL, getDBAdapter }

package/src/mysql.db.ts

@@ -10,7 +10,7 @@ import {
   RunQueryResult,
 } from '@naturalcycles/db-lib'
 import {
-  _filterNullishValues,
+  _filterUndefinedValues,
   _mapKeys,
   _mapValues,
   _Memo,
@@ -136,7 +136,7 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
   async close(): Promise<void> {
     const pool = this.pool()
     // eslint-disable-next-line @typescript-eslint/await-thenable
-    await promisify(pool.end.bind(pool))
+    await promisify(pool.end.bind(pool))()
     this.cfg.logger.log('closed')
   }
 
@@ -191,14 +191,24 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
     _opt: MysqlDBOptions = {},
   ): Promise<RunQueryResult<OUT>> {
     const sql = dbQueryToSQLSelect(q)
-    const rows = await this.runSQL<ROW[]>({ sql })
+    const rows = (await this.runSQL<ROW[]>({ sql })).map(
+      row => _mapKeys(_filterUndefinedValues(row, true), k => mapNameFromMySQL(k)) as any,
+    )
+
+    // edge case where 0 fields are selected
+    if (q._selectedFieldNames?.length === 0) {
+      return {
+        rows: rows.map(_ => ({} as any)),
+      }
+    }
+
     return {
-      rows: rows.map(r => _mapKeys(_filterNullishValues(r, true), k => mapNameFromMySQL(k)) as any),
+      rows,
     }
   }
 
   async runSQL<RESULT>(q: QueryOptions): Promise<RESULT> {
-    if (this.cfg.logSQL) this.cfg.logger.log(q.sql, q.values)
+    if (this.cfg.logSQL) this.cfg.logger.log(...[q.sql, q.values].filter(Boolean))
 
     return await new Promise<RESULT>((resolve, reject) => {
       this.pool().query(q, (err, res) => {
@@ -208,6 +218,21 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
     })
   }
 
+  /**
+   * Allows to run semicolon-separated "SQL file".
+   * E.g "ddl reset script".
+   */
+  async runSQLString(s: string): Promise<void> {
+    const queries = s
+      .split(';')
+      .map(s => s.trim())
+      .filter(Boolean)
+
+    for await (const sql of queries) {
+      await this.runSQL({ sql })
+    }
+  }
+
   override async runQueryCount<ROW extends ObjectWithId>(
     q: DBQuery<ROW>,
     _opt?: CommonDBOptions,
@@ -232,7 +257,7 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
         new Transform({
           objectMode: true,
           transform(row, _encoding, cb) {
-            cb(null, _filterNullishValues(row, true))
+            cb(null, _filterUndefinedValues(row, true))
           },
         }),
       )
@@ -248,13 +273,13 @@ export class MysqlDB extends BaseCommonDB implements CommonDB {
 
     // Stringify object values
     const rows = _rows.map(row =>
-      _mapValues(row, v => {
+      _mapValues(row, (_k, v) => {
         return v && typeof v === 'object' && !Buffer.isBuffer(v) ? JSON.stringify(v) : v
       }),
     )
 
     // inserts are split into multiple sentenses to respect the max_packet_size (1Mb usually)
-    const sqls = insertSQL(table, rows, 'INSERT', this.cfg.logger)
+    const sqls = insertSQL(table, rows, 'REPLACE', this.cfg.logger)
 
     for await (const sql of sqls) {
       await this.runSQL({ sql })

package/src/mysqlKeyValueDB.ts

@@ -0,0 +1,148 @@
+import { Transform } from 'stream'
+import { CommonDBCreateOptions, CommonKeyValueDB, KeyValueDBTuple } from '@naturalcycles/db-lib'
+import { pMap } from '@naturalcycles/js-lib'
+import { ReadableTyped } from '@naturalcycles/nodejs-lib'
+import { QueryOptions } from 'mysql'
+import { MysqlDB, MysqlDBCfg } from './mysql.db'
+
+interface KeyValueObject {
+  id: string
+  v: Buffer
+}
+
+export class MySQLKeyValueDB implements CommonKeyValueDB {
+  constructor(public cfg: MysqlDBCfg) {
+    this.db = new MysqlDB(this.cfg)
+  }
+
+  db: MysqlDB
+
+  async ping(): Promise<void> {
+    await this.db.ping()
+  }
+
+  async close(): Promise<void> {
+    await this.db.close()
+  }
+
+  async createTable(table: string, opt: CommonDBCreateOptions = {}): Promise<void> {
+    if (opt.dropIfExists) await this.dropTable(table)
+
+    // On blob sizes: https://tableplus.com/blog/2019/10/tinyblob-blob-mediumblob-longblob.html
+    // LONGBLOB supports up to 4gb
+    // MEDIUMBLOB up to 16Mb
+    const sql = `create table ${table} (id VARCHAR(64) PRIMARY KEY, v LONGBLOB NOT NULL)`
+    this.db.cfg.logger.log(sql)
+    await this.db.runSQL({ sql })
+  }
+
+  async getByIds(table: string, ids: string[]): Promise<KeyValueDBTuple[]> {
+    if (!ids.length) return []
+
+    const sql = `SELECT id,v FROM ${table} where id in (${ids.map(id => `"${id}"`).join(',')})`
+
+    const rows = await this.db.runSQL<KeyValueObject[]>({ sql })
+
+    return rows.map(({ id, v }) => [id, v])
+  }
+
+  /**
+   * Use with caution!
+   */
+  async dropTable(table: string): Promise<void> {
+    await this.db.runSQL({ sql: `DROP TABLE IF EXISTS ${table}` })
+  }
+
+  async deleteByIds(table: string, ids: string[]): Promise<void> {
+    const sql = `DELETE FROM ${table} WHERE id in (${ids.map(id => `"${id}"`).join(',')})`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(sql)
+    await this.db.runSQL({ sql })
+  }
+
+  async saveBatch(table: string, entries: KeyValueDBTuple[]): Promise<void> {
+    const statements: QueryOptions[] = entries.map(([id, buf]) => {
+      return {
+        sql: `INSERT INTO ${table} (id, v) VALUES (?, ?)`,
+        values: [id, buf],
+      }
+    })
+
+    await pMap(statements, async statement => {
+      if (this.cfg.debug) this.db.cfg.logger.log(statement.sql)
+      await this.db.runSQL(statement)
+    })
+  }
+
+  streamIds(table: string, limit?: number): ReadableTyped<string> {
+    let sql = `SELECT id FROM ${table}`
+    if (limit) sql += ` LIMIT ${limit}`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(`stream: ${sql}`)
+
+    return this.db
+      .pool()
+      .query(sql)
+      .stream()
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform(row, _, cb) {
+            cb(null, row.id)
+          },
+        }),
+      )
+  }
+
+  streamValues(table: string, limit?: number): ReadableTyped<Buffer> {
+    let sql = `SELECT v FROM ${table}`
+    if (limit) sql += ` LIMIT ${limit}`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(`stream: ${sql}`)
+
+    return this.db
+      .pool()
+      .query(sql)
+      .stream()
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform(row, _, cb) {
+            cb(null, row.v)
+          },
+        }),
+      )
+  }
+
+  streamEntries(table: string, limit?: number): ReadableTyped<KeyValueDBTuple> {
+    let sql = `SELECT id,v FROM ${table}`
+    if (limit) sql += ` LIMIT ${limit}`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(`stream: ${sql}`)
+
+    return this.db
+      .pool()
+      .query(sql)
+      .stream()
+      .pipe(
+        new Transform({
+          objectMode: true,
+          transform(row, _, cb) {
+            cb(null, [row.id, row.v])
+          },
+        }),
+      )
+  }
+
+  async beginTransaction(): Promise<void> {
+    await this.db.runSQL({ sql: `BEGIN TRANSACTION` })
+  }
+
+  async endTransaction(): Promise<void> {
+    await this.db.runSQL({ sql: `END TRANSACTION` })
+  }
+
+  async count(table: string): Promise<number> {
+    const sql = `SELECT count(*) as cnt FROM ${table}`
+    if (this.cfg.logSQL) this.db.cfg.logger.log(sql)
+
+    const rows = await this.db.runSQL<{ cnt: number }[]>({ sql })
+    return rows[0]!.cnt
+  }
+}

package/src/query.util.ts

@@ -16,7 +16,7 @@ export function dbQueryToSQLSelect(q: DBQuery<any>): string {
   tokens.push(...whereTokens(q))
 
   // order
-  tokens.push(...orderTokens(q))
+  tokens.push(...groupOrderTokens(q))
 
   // offset/limit
   tokens.push(...offsetLimitTokens(q))
@@ -41,7 +41,7 @@ export function dbQueryToSQLDelete(q: DBQuery<any>): string {
  */
 export function insertSQL(
   table: string,
-  records: Record<any, any>[],
+  rows: Record<any, any>[],
   verb: 'INSERT' | 'REPLACE' = 'INSERT',
   logger: CommonLogger = console,
 ): string[] {
@@ -49,8 +49,8 @@ export function insertSQL(
   // VALUES (value1, value2, value3, ...);
 
   // eslint-disable-next-line unicorn/no-array-reduce
-  const fieldSet = records.reduce((set: Set<string>, rec) => {
-    Object.keys(rec).forEach(field => set.add(field))
+  const fieldSet = rows.reduce((set: Set<string>, row) => {
+    Object.keys(row).forEach(field => set.add(field))
     return set
   }, new Set<string>())
   const fields = [...fieldSet]
@@ -63,7 +63,7 @@ export function insertSQL(
     `VALUES\n`,
   ].join(' ')
 
-  const valueRows = records.map(rec => {
+  const valueRows = rows.map(rec => {
     return `(` + fields.map(k => mysql.escape(rec[k])).join(',') + `)`
   })
 
@@ -155,7 +155,13 @@ function selectTokens(q: DBQuery): string[] {
   }
 
   // We don't do `escapeId` cause it'll ruin e.g SELECT `count *` FROM ...
-  return [`SELECT`, fields.map(f => mapNameToMySQL(f)).join(', '), `FROM`, mysql.escapeId(q.table)]
+  return [
+    `SELECT`,
+    q._distinct && ('DISTINCT' as any),
+    fields.map(f => mapNameToMySQL(f)).join(', '),
+    `FROM`,
+    mysql.escapeId(q.table),
+  ].filter(Boolean)
 }
 
 function offsetLimitTokens(q: DBQuery<any>): string[] {
@@ -173,12 +179,21 @@ function offsetLimitTokens(q: DBQuery<any>): string[] {
   return tokens
 }
 
-function orderTokens(q: DBQuery): string[] {
-  if (!q._orders.length) return []
-  return [
-    `ORDER BY`,
-    q._orders.map(o => `\`${o.name}\` ${o.descending ? 'DESC' : 'ASC'}`).join(', '),
-  ]
+function groupOrderTokens(q: DBQuery): string[] {
+  const t: string[] = []
+
+  if (q._groupByFieldNames?.length) {
+    t.push(`GROUP BY`, q._groupByFieldNames.map(c => `\`${c}\``).join(', '))
+  }
+
+  if (q._orders.length) {
+    t.push(
+      `ORDER BY`,
+      q._orders.map(o => `\`${o.name}\` ${o.descending ? 'DESC' : 'ASC'}`).join(', '),
+    )
+  }
+
+  return t
 }
 
 const OP_MAP: Partial<Record<DBQueryFilterOperator, string>> = {

package/src/schema/mysql.schema.util.ts

@@ -106,7 +106,7 @@ export function mysqlTableStatsToJsonSchemaField<T = any>(
       s.properties[name] = { instanceof: 'Buffer' }
     } else if (t.startsWith('tinyint') || t.includes('(1)')) {
       s.properties[name] = { type: 'boolean' } as JsonSchemaBoolean
-    } else if (t.startsWith('int(')) {
+    } else if (t === 'int' || t.startsWith('int(')) {
       s.properties[name] = { type: 'integer' } as JsonSchemaNumber
     } else if (t.startsWith('float')) {
       s.properties[name] = { type: 'number' } as JsonSchemaNumber