@naturalcycles/mysql-lib 3.8.0

package/CHANGELOG.md

@@ -0,0 +1,198 @@
+# [3.8.0](https://github.com/NaturalCycles/mysql-lib/compare/v3.7.1...v3.8.0) (2021-10-04)
+
+
+### Features
+
+* modernize ([6e8f0a2](https://github.com/NaturalCycles/mysql-lib/commit/6e8f0a254a67902cbc185f034f58cd4f6885449d))
+
+## [3.7.1](https://github.com/NaturalCycles/mysql-lib/compare/v3.7.0...v3.7.1) (2020-06-03)
+
+
+### Bug Fixes
+
+* modernize ([517f695](https://github.com/NaturalCycles/mysql-lib/commit/517f6952feb15059730f4f7443a295bf596053fc))
+
+# [3.7.0](https://github.com/NaturalCycles/mysql-lib/compare/v3.6.0...v3.7.0) (2020-04-08)
+
+
+### Features
+
+* modernize ([e0e2890](https://github.com/NaturalCycles/mysql-lib/commit/e0e2890cb31e88fcbdae63c417d6cc5c6a52d4e8))
+
+# [3.6.0](https://github.com/NaturalCycles/mysql-lib/compare/v3.5.2...v3.6.0) (2020-03-01)
+
+
+### Features
+
+* deps ([008aa10](https://github.com/NaturalCycles/mysql-lib/commit/008aa10d059734d76c2354553a8dc35c8117b3b2))
+
+## [3.5.2](https://github.com/NaturalCycles/mysql-lib/compare/v3.5.1...v3.5.2) (2019-11-20)
+
+
+### Bug Fixes
+
+* BIT typecast to boolean correctly ([0bc815e](https://github.com/NaturalCycles/mysql-lib/commit/0bc815e025a33ab9fae5405c8b97dd53aae392ca))
+
+## [3.5.1](https://github.com/NaturalCycles/mysql-lib/compare/v3.5.0...v3.5.1) (2019-11-20)
+
+
+### Bug Fixes
+
+* correctly type-cast booleans with value=NULL ([e208b4c](https://github.com/NaturalCycles/mysql-lib/commit/e208b4ca16f700b6ed7bf95c872f68f934894992))
+
+# [3.5.0](https://github.com/NaturalCycles/mysql-lib/compare/v3.4.0...v3.5.0) (2019-11-09)
+
+
+### Features
+
+* implement getDBAdapter. Allow empty (default) configuration ([bcf12dc](https://github.com/NaturalCycles/mysql-lib/commit/bcf12dc9360f525749f9bb10965046b4c56d849a))
+
+# [3.4.0](https://github.com/NaturalCycles/mysql-lib/compare/v3.3.1...v3.4.0) (2019-11-09)
+
+
+### Features
+
+* auto JSON.stringify objects (and arrays) ([0d9aaed](https://github.com/NaturalCycles/mysql-lib/commit/0d9aaedf8f2920a51dacf13b6723eb98c61b34c3))
+
+## [3.3.1](https://github.com/NaturalCycles/mysql-lib/compare/v3.3.0...v3.3.1) (2019-11-09)
+
+
+### Bug Fixes
+
+* use LONGBLOB for binary columns ([ae4facc](https://github.com/NaturalCycles/mysql-lib/commit/ae4facc76bc03b8aafa5141199a98bef530f6913))
+
+# [3.3.0](https://github.com/NaturalCycles/mysql-lib/compare/v3.2.1...v3.3.0) (2019-11-09)
+
+
+### Features
+
+* mapNameToMySQL ([c4bf40c](https://github.com/NaturalCycles/mysql-lib/commit/c4bf40cd6aefaf78219a2588ec803cf74004bc07))
+
+## [3.2.1](https://github.com/NaturalCycles/mysql-lib/compare/v3.2.0...v3.2.1) (2019-11-09)
+
+
+### Bug Fixes
+
+* createTable() ([7983085](https://github.com/NaturalCycles/mysql-lib/commit/7983085017562a4e4a8f7dd38eb41ae6b9fc30ed))
+
+# [3.2.0](https://github.com/NaturalCycles/mysql-lib/compare/v3.1.0...v3.2.0) (2019-11-09)
+
+
+### Bug Fixes
+
+* emoji support ([1262fc4](https://github.com/NaturalCycles/mysql-lib/commit/1262fc4d9ad39c818833b5cfc31b3a65979cdea0))
+
+
+### Features
+
+* auto split long sql queries ([c7c780f](https://github.com/NaturalCycles/mysql-lib/commit/c7c780f295d7ded9f418509f3160a512ec54c7e7))
+
+# [3.1.0](https://github.com/NaturalCycles/mysql-lib/compare/v3.0.0...v3.1.0) (2019-11-08)
+
+
+### Features
+
+* commonSchemaToMySQLDDL ([09168c5](https://github.com/NaturalCycles/mysql-lib/commit/09168c563cbb629ca47121a1aac6a125dd23c144))
+* implement getTables(), getTableSchema() ([20a7746](https://github.com/NaturalCycles/mysql-lib/commit/20a7746c8c4c88dfca8365d1e8d2decce3f47efa))
+
+# [3.0.0](https://github.com/NaturalCycles/mysql-lib/compare/v2.0.2...v3.0.0) (2019-11-02)
+
+
+### Features
+
+* adapt to db-lib@3 ([7d13f54](https://github.com/NaturalCycles/mysql-lib/commit/7d13f540c92dd59b014d1a502c1bff937fad567f))
+
+
+### BREAKING CHANGES
+
+* ^^^
+
+## [2.0.2](https://github.com/NaturalCycles/mysql-lib/compare/v2.0.1...v2.0.2) (2019-10-20)
+
+
+### Bug Fixes
+
+* adopt to db-lib ([6da7d78](https://github.com/NaturalCycles/mysql-lib/commit/6da7d78))
+
+## [2.0.1](https://github.com/NaturalCycles/mysql-lib/compare/v2.0.0...v2.0.1) (2019-10-19)
+
+
+### Bug Fixes
+
+* use Readable ([a0d09c1](https://github.com/NaturalCycles/mysql-lib/commit/a0d09c1))
+
+# [2.0.0](https://github.com/NaturalCycles/mysql-lib/compare/v1.7.1...v2.0.0) (2019-10-18)
+
+
+### Features
+
+* implement CommonDB 2.0 ([9de0e9f](https://github.com/NaturalCycles/mysql-lib/commit/9de0e9f))
+
+
+### BREAKING CHANGES
+
+* ^^^
+
+## [1.7.1](https://github.com/NaturalCycles/mysql-lib/compare/v1.7.0...v1.7.1) (2019-10-18)
+
+
+### Bug Fixes
+
+* pin @types/hapi__joi ([3e3cf5a](https://github.com/NaturalCycles/mysql-lib/commit/3e3cf5a))
+
+# [1.7.0](https://github.com/NaturalCycles/mysql-lib/compare/v1.6.0...v1.7.0) (2019-09-30)
+
+
+### Features
+
+* tiny change to use pool.query inst of getConnection() ([6ffe1b4](https://github.com/NaturalCycles/mysql-lib/commit/6ffe1b4))
+
+# [1.6.0](https://github.com/NaturalCycles/mysql-lib/compare/v1.5.0...v1.6.0) (2019-09-30)
+
+
+### Features
+
+* offset ([78710a1](https://github.com/NaturalCycles/mysql-lib/commit/78710a1))
+
+# [1.5.0](https://github.com/NaturalCycles/mysql-lib/compare/v1.4.0...v1.5.0) (2019-09-30)
+
+
+### Features
+
+* testing simpler streamSQL impl ([0887aa3](https://github.com/NaturalCycles/mysql-lib/commit/0887aa3))
+
+# [1.4.0](https://github.com/NaturalCycles/mysql-lib/compare/v1.3.0...v1.4.0) (2019-09-30)
+
+
+### Features
+
+* stream to use it's own separate connection ([43615f3](https://github.com/NaturalCycles/mysql-lib/commit/43615f3))
+
+# [1.3.0](https://github.com/NaturalCycles/mysql-lib/compare/v1.2.0...v1.3.0) (2019-09-30)
+
+
+### Features
+
+* cfg.debugConnections ([70218ba](https://github.com/NaturalCycles/mysql-lib/commit/70218ba))
+
+# [1.2.0](https://github.com/NaturalCycles/mysql-lib/compare/v1.1.0...v1.2.0) (2019-09-28)
+
+
+### Features
+
+* add BIT cast to boolean ([899cce4](https://github.com/NaturalCycles/mysql-lib/commit/899cce4))
+
+# [1.1.0](https://github.com/NaturalCycles/mysql-lib/compare/v1.0.0...v1.1.0) (2019-09-21)
+
+
+### Features
+
+* modernize, implement saveBatch ([5595b6a](https://github.com/NaturalCycles/mysql-lib/commit/5595b6a))
+
+# 1.0.0 (2019-06-03)
+
+
+### Features
+
+* first version ([02620fb](https://github.com/NaturalCycles/mysql-lib/commit/02620fb))
+* init project by create-module ([eee50b7](https://github.com/NaturalCycles/mysql-lib/commit/eee50b7))

package/dist/dbAdapter.d.ts

@@ -0,0 +1,2 @@
+import { MysqlDB } from './mysql.db';
+export declare function getDBAdapter(cfgStr?: string): MysqlDB;

package/dist/dbAdapter.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDBAdapter = void 0;
+const mysql_db_1 = require("./mysql.db");
+function getDBAdapter(cfgStr = '{}') {
+    const cfg = JSON.parse(cfgStr);
+    return new mysql_db_1.MysqlDB(cfg);
+}
+exports.getDBAdapter = getDBAdapter;

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+import { getDBAdapter } from './dbAdapter';
+import { MysqlDB, MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions } from './mysql.db';
+import { jsonSchemaToMySQLDDL } from './schema/mysql.schema.util';
+export type { MysqlDBCfg, MysqlDBOptions, MysqlDBSaveOptions };
+export { MysqlDB, jsonSchemaToMySQLDDL, getDBAdapter };

package/dist/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDBAdapter = exports.jsonSchemaToMySQLDDL = exports.MysqlDB = void 0;
+const dbAdapter_1 = require("./dbAdapter");
+Object.defineProperty(exports, "getDBAdapter", { enumerable: true, get: function () { return dbAdapter_1.getDBAdapter; } });
+const mysql_db_1 = require("./mysql.db");
+Object.defineProperty(exports, "MysqlDB", { enumerable: true, get: function () { return mysql_db_1.MysqlDB; } });
+const mysql_schema_util_1 = require("./schema/mysql.schema.util");
+Object.defineProperty(exports, "jsonSchemaToMySQLDDL", { enumerable: true, get: function () { return mysql_schema_util_1.jsonSchemaToMySQLDDL; } });

package/dist/mysql.db.d.ts

@@ -0,0 +1,58 @@
+import { BaseCommonDB, BaseDBEntity, CommonDB, CommonDBCreateOptions, CommonDBOptions, CommonDBSaveOptions, DBQuery, ObjectWithId, RunQueryResult } from '@naturalcycles/db-lib';
+import { JsonSchemaObject, JsonSchemaRootObject } from '@naturalcycles/js-lib';
+import { ReadableTyped } from '@naturalcycles/nodejs-lib';
+import { Connection, Pool, PoolConfig, PoolConnection, QueryOptions } from 'mysql';
+export interface MysqlDBOptions extends CommonDBOptions {
+}
+export interface MysqlDBSaveOptions extends CommonDBSaveOptions {
+}
+/**
+ * @default false / undefined
+ */
+export interface MysqlDBCfg extends PoolConfig {
+    /**
+     * If true - will log all produced SQL queries.
+     */
+    logSQL?: boolean;
+    /**
+     * If true - will emit logs of connection events.
+     */
+    debugConnections?: boolean;
+}
+export declare class MysqlDB extends BaseCommonDB implements CommonDB {
+    constructor(cfg?: MysqlDBCfg);
+    cfg: MysqlDBCfg;
+    ping(): Promise<void>;
+    pool(): Pool;
+    close(): Promise<void>;
+    /**
+     * Be careful to always call `con.release()` when you get connection with this method.
+     */
+    getConnection(): Promise<PoolConnection>;
+    /**
+     * Manually create a single (not pool) connection.
+     * Be careful to manage this connection yourself (open, release, etc).
+     */
+    createSingleConnection(): Promise<Connection>;
+    getByIds<ROW extends ObjectWithId>(table: string, ids: string[], opt?: MysqlDBOptions): Promise<ROW[]>;
+    runQuery<ROW extends ObjectWithId, OUT = ROW>(q: DBQuery<ROW>, _opt?: MysqlDBOptions): Promise<RunQueryResult<OUT>>;
+    runSQL<RESULT>(q: QueryOptions): Promise<RESULT>;
+    runQueryCount<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
+    streamQuery<ROW extends ObjectWithId, OUT = ROW>(q: DBQuery<ROW>, _opt?: MysqlDBOptions): ReadableTyped<OUT>;
+    saveBatch<ROW extends BaseDBEntity>(table: string, _rows: ROW[], _opt?: MysqlDBSaveOptions): Promise<void>;
+    /**
+     * Limitation: always returns [], regardless of which rows are actually deleted
+     */
+    deleteByIds(table: string, ids: string[], _opt?: MysqlDBOptions): Promise<number>;
+    deleteByQuery<ROW extends ObjectWithId>(q: DBQuery<ROW>, _opt?: CommonDBOptions): Promise<number>;
+    /**
+     * Use with caution!
+     */
+    dropTable(table: string): Promise<void>;
+    /**
+     * dropIfExists=true needed as a safety check
+     */
+    createTable(table: string, schema: JsonSchemaObject, opt?: CommonDBCreateOptions): Promise<void>;
+    getTables(): Promise<string[]>;
+    getTableSchema<ROW extends ObjectWithId>(table: string): Promise<JsonSchemaRootObject<ROW>>;
+}

package/dist/mysql.db.js

@@ -0,0 +1,214 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MysqlDB = void 0;
+const tslib_1 = require("tslib");
+const stream_1 = require("stream");
+const util_1 = require("util");
+const db_lib_1 = require("@naturalcycles/db-lib");
+const js_lib_1 = require("@naturalcycles/js-lib");
+const nodejs_lib_1 = require("@naturalcycles/nodejs-lib");
+const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
+const mysql = require("mysql");
+const query_util_1 = require("./query.util");
+const mysql_schema_util_1 = require("./schema/mysql.schema.util");
+const log = (0, nodejs_lib_1.Debug)('nc:mysql-lib');
+const BOOLEAN_TYPES = new Set(['TINY', 'TINYINT', 'INT']);
+const BOOLEAN_BIT_TYPES = new Set(['BIT']);
+const typeCast = (field, next) => {
+    // cast to boolean
+    if (field.length === 1) {
+        if (BOOLEAN_BIT_TYPES.has(field.type)) {
+            const b = field.buffer();
+            if (b === null || b === undefined)
+                return undefined; // null = undefined
+            return b[0] === 1; // 1 = true, 0 = false
+        }
+        if (BOOLEAN_TYPES.has(field.type)) {
+            const s = field.string();
+            // console.log(field.name, field.type, s, s?.charCodeAt(0))
+            if (s === null || s === undefined)
+                return undefined; // null = undefined
+            return s === '1'; // 1 = true, 0 = false
+        }
+    }
+    return next();
+};
+class MysqlDB extends db_lib_1.BaseCommonDB {
+    constructor(cfg = {}) {
+        super();
+        this.cfg = {
+            typeCast,
+            // charset: 'utf8mb4', // for emoji support
+            // host: 'localhost',
+            // user: MYSQL_USER,
+            // password: MYSQL_PW,
+            // database: MYSQL_DB,
+            ...cfg,
+        };
+    }
+    async ping() {
+        const con = await this.getConnection();
+        con.release();
+    }
+    pool() {
+        const pool = mysql.createPool(this.cfg);
+        const { host, database } = this.cfg;
+        log(`connected to ${(0, colors_1.white)(host + '/' + database)}`);
+        if (this.cfg.debugConnections) {
+            pool.on('acquire', con => {
+                log(`acquire(${con.threadId})`);
+            });
+            pool.on('connection', con => {
+                log(`connection(${con.threadId})`);
+            });
+            pool.on('enqueue', () => {
+                log(`enqueue`);
+            });
+            pool.on('release', con => {
+                log(`release(${con.threadId})`);
+            });
+        }
+        pool.on('error', err => {
+            log.error(`error`, err);
+        });
+        return pool;
+    }
+    async close() {
+        const pool = this.pool();
+        // eslint-disable-next-line @typescript-eslint/await-thenable
+        await (0, util_1.promisify)(pool.end.bind(pool));
+    }
+    /**
+     * Be careful to always call `con.release()` when you get connection with this method.
+     */
+    async getConnection() {
+        const pool = this.pool();
+        return await (0, util_1.promisify)(pool.getConnection.bind(pool))();
+    }
+    /**
+     * Manually create a single (not pool) connection.
+     * Be careful to manage this connection yourself (open, release, etc).
+     */
+    async createSingleConnection() {
+        const con = mysql.createConnection(this.cfg);
+        await (0, util_1.promisify)(con.connect.bind(con))();
+        const { threadId } = con;
+        if (this.cfg.debugConnections) {
+            log(`createSingleConnection(${threadId})`);
+            con.on('connect', () => log(`createSingleConnection(${threadId}).connect`));
+            con.on('drain', () => log(`createSingleConnection(${threadId}).drain`));
+            con.on('enqueue', () => log(`createSingleConnection(${threadId}).enqueue`));
+            con.on('end', () => log(`createSingleConnection(${threadId}).end`));
+        }
+        con.on('error', err => {
+            log.error(`createSingleConnection(${threadId}).error`, err);
+        });
+        return con;
+    }
+    // GET
+    async getByIds(table, ids, opt = {}) {
+        if (!ids.length)
+            return [];
+        const q = new db_lib_1.DBQuery(table).filterEq('id', ids);
+        const { rows } = await this.runQuery(q, opt);
+        return rows.map(r => (0, js_lib_1._mapKeys)(r, k => (0, mysql_schema_util_1.mapNameFromMySQL)(k)));
+    }
+    // QUERY
+    async runQuery(q, _opt = {}) {
+        const sql = (0, query_util_1.dbQueryToSQLSelect)(q);
+        const rows = await this.runSQL({ sql });
+        return {
+            rows: rows.map(r => (0, js_lib_1._mapKeys)((0, js_lib_1._filterNullishValues)(r, true), k => (0, mysql_schema_util_1.mapNameFromMySQL)(k))),
+        };
+    }
+    async runSQL(q) {
+        if (this.cfg.logSQL)
+            log(q.sql, q.values);
+        return await new Promise((resolve, reject) => {
+            this.pool().query(q, (err, res) => {
+                if (err)
+                    return reject(err);
+                resolve(res);
+            });
+        });
+    }
+    async runQueryCount(q, _opt) {
+        const { rows } = await this.runQuery(q.select(['count(*) as _count']));
+        return rows[0]._count;
+    }
+    streamQuery(q, _opt = {}) {
+        const sql = (0, query_util_1.dbQueryToSQLSelect)(q);
+        if (this.cfg.logSQL)
+            log(`stream: ${sql}`);
+        // return this.streamSQL(sql, opt)
+        return this.pool()
+            .query(sql)
+            .stream()
+            .pipe(new stream_1.Transform({
+            objectMode: true,
+            transform(row, _encoding, cb) {
+                cb(null, (0, js_lib_1._filterNullishValues)(row, true));
+            },
+        }));
+    }
+    // SAVE
+    async saveBatch(table, _rows, _opt) {
+        if (!_rows.length)
+            return;
+        // Stringify object values
+        const rows = _rows.map(row => (0, js_lib_1._mapValues)(row, v => {
+            return v && typeof v === 'object' && !Buffer.isBuffer(v) ? JSON.stringify(v) : v;
+        }));
+        // inserts are split into multiple sentenses to respect the max_packet_size (1Mb usually)
+        const sqls = (0, query_util_1.insertSQL)(table, rows);
+        for await (const sql of sqls) {
+            await this.runSQL({ sql });
+        }
+    }
+    // DELETE
+    /**
+     * Limitation: always returns [], regardless of which rows are actually deleted
+     */
+    async deleteByIds(table, ids, _opt) {
+        if (!ids.length)
+            return 0;
+        const sql = (0, query_util_1.dbQueryToSQLDelete)(new db_lib_1.DBQuery(table).filterEq('id', ids));
+        const res = await this.runSQL({ sql });
+        return res.affectedRows;
+    }
+    async deleteByQuery(q, _opt) {
+        const sql = (0, query_util_1.dbQueryToSQLDelete)(q);
+        const res = await this.runSQL({ sql });
+        return res.affectedRows;
+    }
+    /**
+     * Use with caution!
+     */
+    async dropTable(table) {
+        await this.runSQL({ sql: `DROP TABLE IF EXISTS ${table}` });
+    }
+    /**
+     * dropIfExists=true needed as a safety check
+     */
+    async createTable(table, schema, opt = {}) {
+        if (opt.dropIfExists)
+            await this.dropTable(table);
+        const sql = (0, mysql_schema_util_1.jsonSchemaToMySQLDDL)(table, schema);
+        await this.runSQL({ sql });
+    }
+    async getTables() {
+        return (await this.runSQL({ sql: `show tables` }))
+            .map(r => Object.values(r)[0])
+            .filter(Boolean);
+    }
+    async getTableSchema(table) {
+        const stats = await this.runSQL({
+            sql: `describe ${mysql.escapeId(table)}`,
+        });
+        return (0, mysql_schema_util_1.mysqlTableStatsToJsonSchemaField)(table, stats);
+    }
+}
+(0, tslib_1.__decorate)([
+    (0, js_lib_1._Memo)()
+], MysqlDB.prototype, "pool", null);
+exports.MysqlDB = MysqlDB;

package/dist/query.util.d.ts

@@ -0,0 +1,11 @@
+import { DBQuery } from '@naturalcycles/db-lib';
+import { QueryOptions } from 'mysql';
+export declare function dbQueryToSQLSelect(q: DBQuery<any>): string;
+export declare function dbQueryToSQLDelete(q: DBQuery<any>): string;
+/**
+ * Returns array of sql statements to respect the max sql size.
+ */
+export declare function insertSQL(table: string, records: Record<any, any>[], verb?: 'INSERT' | 'REPLACE'): string[];
+export declare function insertSQLSingle(table: string, record: Record<any, any>): string;
+export declare function insertSQLSetSingle(table: string, record: Record<any, any>): QueryOptions;
+export declare function dbQueryToSQLUpdate(q: DBQuery<any>, record: Record<any, any>): string;

package/dist/query.util.js

@@ -0,0 +1,177 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dbQueryToSQLUpdate = exports.insertSQLSetSingle = exports.insertSQLSingle = exports.insertSQL = exports.dbQueryToSQLDelete = exports.dbQueryToSQLSelect = void 0;
+const js_lib_1 = require("@naturalcycles/js-lib");
+const colors_1 = require("@naturalcycles/nodejs-lib/dist/colors");
+const mysql = require("mysql");
+const mysql_schema_util_1 = require("./schema/mysql.schema.util");
+const MAX_PACKET_SIZE = 1024 * 1024; // 1Mb
+const MAX_ROW_SIZE = 800 * 1024; // 1Mb - margin
+function dbQueryToSQLSelect(q) {
+    const tokens = selectTokens(q);
+    // filters
+    tokens.push(...whereTokens(q));
+    // order
+    tokens.push(...orderTokens(q));
+    // offset/limit
+    tokens.push(...offsetLimitTokens(q));
+    return tokens.join(' ');
+}
+exports.dbQueryToSQLSelect = dbQueryToSQLSelect;
+function dbQueryToSQLDelete(q) {
+    const tokens = [`DELETE FROM`, mysql.escapeId(q.table)];
+    // filters
+    tokens.push(...whereTokens(q));
+    // offset/limit
+    tokens.push(...offsetLimitTokens(q));
+    return tokens.join(' ');
+}
+exports.dbQueryToSQLDelete = dbQueryToSQLDelete;
+/**
+ * Returns array of sql statements to respect the max sql size.
+ */
+function insertSQL(table, records, verb = 'INSERT') {
+    // INSERT INTO table_name (column1, column2, column3, ...)
+    // VALUES (value1, value2, value3, ...);
+    // eslint-disable-next-line unicorn/no-array-reduce
+    const fieldSet = records.reduce((set, rec) => {
+        Object.keys(rec).forEach(field => set.add(field));
+        return set;
+    }, new Set());
+    const fields = [...fieldSet];
+    const start = [
+        verb,
+        `INTO`,
+        mysql.escapeId(table),
+        `(` + [...fields].map(f => mysql.escapeId((0, mysql_schema_util_1.mapNameToMySQL)(f))).join(',') + `)`,
+        `VALUES\n`,
+    ].join(' ');
+    const valueRows = records.map(rec => {
+        return `(` + fields.map(k => mysql.escape(rec[k])).join(',') + `)`;
+    });
+    const full = start + valueRows.join(',\n');
+    if (full.length < MAX_PACKET_SIZE)
+        return [full];
+    const sqls = [];
+    let sql;
+    valueRows.forEach(vrow => {
+        if (!sql) {
+            sql = start + vrow;
+        }
+        else {
+            if (sql.length + vrow.length >= MAX_ROW_SIZE) {
+                sqls.push(sql);
+                sql = start + vrow; // reset
+            }
+            else {
+                sql += ',\n ' + vrow; // add
+            }
+        }
+    });
+    if (sql) {
+        sqls.push(sql); // last one
+    }
+    console.log(`${(0, colors_1.white)(table)} large sql query (${(0, colors_1.yellow)((0, js_lib_1._hb)(full.length))}) was split into ${(0, colors_1.yellow)(sqls.length)} chunks`);
+    return sqls;
+    // todo: handle "upsert" later
+}
+exports.insertSQL = insertSQL;
+function insertSQLSingle(table, record) {
+    // INSERT INTO table_name (column1, column2, column3, ...)
+    // VALUES (value1, value2, value3, ...);
+    const tokens = [
+        `INSERT INTO`,
+        mysql.escapeId(table),
+        `(` +
+            Object.keys(record)
+                .map(f => mysql.escapeId((0, mysql_schema_util_1.mapNameToMySQL)(f)))
+                .join(',') +
+            `)`,
+        `\nVALUES`,
+        `(` +
+            Object.values(record)
+                .map(v => mysql.escapeId(v))
+                .join(',') +
+            `)`,
+    ];
+    return tokens.join(' ');
+}
+exports.insertSQLSingle = insertSQLSingle;
+function insertSQLSetSingle(table, record) {
+    return {
+        sql: `INSERT INTO ${mysql.escapeId(table)} SET ?`,
+        values: [record],
+    };
+}
+exports.insertSQLSetSingle = insertSQLSetSingle;
+function dbQueryToSQLUpdate(q, record) {
+    // var sql = mysql.format('UPDATE posts SET modified = ? WHERE id = ?', [CURRENT_TIMESTAMP, 42]);
+    const tokens = [
+        `UPDATE`,
+        mysql.escapeId(q.table),
+        `SET`,
+        Object.keys(record)
+            .map(f => mysql.escapeId((0, mysql_schema_util_1.mapNameToMySQL)(f)) + ' = ?')
+            .join(', '),
+        ...whereTokens(q),
+    ];
+    return mysql.format(tokens.join(' '), Object.values(record));
+}
+exports.dbQueryToSQLUpdate = dbQueryToSQLUpdate;
+function selectTokens(q) {
+    let fields = ['*'];
+    if (q._selectedFieldNames) {
+        fields = q._selectedFieldNames.length ? q._selectedFieldNames : ['id'];
+    }
+    // We don't do `escapeId` cause it'll ruin e.g SELECT `count *` FROM ...
+    return [`SELECT`, fields.map(f => (0, mysql_schema_util_1.mapNameToMySQL)(f)).join(', '), `FROM`, mysql.escapeId(q.table)];
+}
+function offsetLimitTokens(q) {
+    const tokens = [];
+    if (q._limitValue) {
+        tokens.push(`LIMIT`, String(q._limitValue));
+        // In SQL OFFSET is only allowed if LIMIT is set
+        if (q._offsetValue) {
+            tokens.push(`OFFSET`, String(q._offsetValue));
+        }
+    }
+    return tokens;
+}
+function orderTokens(q) {
+    if (!q._orders.length)
+        return [];
+    return [
+        `ORDER BY`,
+        q._orders.map(o => `\`${o.name}\` ${o.descending ? 'DESC' : 'ASC'}`).join(', '),
+    ];
+}
+const OP_MAP = {
+    '==': '=',
+};
+function whereTokens(q) {
+    if (!q._filters.length)
+        return [];
+    return [
+        `WHERE`,
+        q._filters
+            .map(f => {
+            if (f.val === null || f.val === undefined) {
+                // special treatment
+                return [
+                    mysql.escapeId((0, mysql_schema_util_1.mapNameToMySQL)(f.name)),
+                    f.op === '==' ? 'IS NULL' : 'IS NOT NULL',
+                ].join(' ');
+            }
+            if (Array.isArray(f.val)) {
+                // special case for arrays
+                return `${mysql.escapeId((0, mysql_schema_util_1.mapNameToMySQL)(f.name))} IN (${mysql.escape(f.val)})`;
+            }
+            return [
+                mysql.escapeId((0, mysql_schema_util_1.mapNameToMySQL)(f.name)),
+                OP_MAP[f.op] || f.op,
+                mysql.escape(f.val),
+            ].join(' ');
+        })
+            .join(' AND '),
+    ];
+}