@naturalcycles/mysql-lib 3.8.0

package/src/query.util.ts

@@ -0,0 +1,216 @@
+import { DBQuery } from '@naturalcycles/db-lib'
+import { DBQueryFilterOperator } from '@naturalcycles/db-lib/src/query/dbQuery'
+import { _hb } from '@naturalcycles/js-lib'
+import { white, yellow } from '@naturalcycles/nodejs-lib/dist/colors'
+import { QueryOptions } from 'mysql'
+import * as mysql from 'mysql'
+import { mapNameToMySQL } from './schema/mysql.schema.util'
+
+const MAX_PACKET_SIZE = 1024 * 1024 // 1Mb
+const MAX_ROW_SIZE = 800 * 1024 // 1Mb - margin
+
+export function dbQueryToSQLSelect(q: DBQuery<any>): string {
+  const tokens = selectTokens(q)
+
+  // filters
+  tokens.push(...whereTokens(q))
+
+  // order
+  tokens.push(...orderTokens(q))
+
+  // offset/limit
+  tokens.push(...offsetLimitTokens(q))
+
+  return tokens.join(' ')
+}
+
+export function dbQueryToSQLDelete(q: DBQuery<any>): string {
+  const tokens = [`DELETE FROM`, mysql.escapeId(q.table)]
+
+  // filters
+  tokens.push(...whereTokens(q))
+
+  // offset/limit
+  tokens.push(...offsetLimitTokens(q))
+
+  return tokens.join(' ')
+}
+
+/**
+ * Returns array of sql statements to respect the max sql size.
+ */
+export function insertSQL(
+  table: string,
+  records: Record<any, any>[],
+  verb: 'INSERT' | 'REPLACE' = 'INSERT',
+): string[] {
+  // INSERT INTO table_name (column1, column2, column3, ...)
+  // VALUES (value1, value2, value3, ...);
+
+  // eslint-disable-next-line unicorn/no-array-reduce
+  const fieldSet = records.reduce((set: Set<string>, rec) => {
+    Object.keys(rec).forEach(field => set.add(field))
+    return set
+  }, new Set<string>())
+  const fields = [...fieldSet]
+
+  const start = [
+    verb,
+    `INTO`,
+    mysql.escapeId(table),
+    `(` + [...fields].map(f => mysql.escapeId(mapNameToMySQL(f))).join(',') + `)`,
+    `VALUES\n`,
+  ].join(' ')
+
+  const valueRows = records.map(rec => {
+    return `(` + fields.map(k => mysql.escape(rec[k])).join(',') + `)`
+  })
+
+  const full = start + valueRows.join(',\n')
+
+  if (full.length < MAX_PACKET_SIZE) return [full]
+
+  const sqls: string[] = []
+  let sql: string | undefined
+
+  valueRows.forEach(vrow => {
+    if (!sql) {
+      sql = start + vrow
+    } else {
+      if (sql.length + vrow.length >= MAX_ROW_SIZE) {
+        sqls.push(sql)
+        sql = start + vrow // reset
+      } else {
+        sql += ',\n ' + vrow // add
+      }
+    }
+  })
+
+  if (sql) {
+    sqls.push(sql) // last one
+  }
+
+  console.log(
+    `${white(table)} large sql query (${yellow(_hb(full.length))}) was split into ${yellow(
+      sqls.length,
+    )} chunks`,
+  )
+
+  return sqls
+
+  // todo: handle "upsert" later
+}
+
+export function insertSQLSingle(table: string, record: Record<any, any>): string {
+  // INSERT INTO table_name (column1, column2, column3, ...)
+  // VALUES (value1, value2, value3, ...);
+
+  const tokens = [
+    `INSERT INTO`,
+    mysql.escapeId(table),
+    `(` +
+      Object.keys(record)
+        .map(f => mysql.escapeId(mapNameToMySQL(f)))
+        .join(',') +
+      `)`,
+    `\nVALUES`,
+    `(` +
+      Object.values(record)
+        .map(v => mysql.escapeId(v))
+        .join(',') +
+      `)`,
+  ]
+
+  return tokens.join(' ')
+}
+
+export function insertSQLSetSingle(table: string, record: Record<any, any>): QueryOptions {
+  return {
+    sql: `INSERT INTO ${mysql.escapeId(table)} SET ?`,
+    values: [record],
+  }
+}
+
+export function dbQueryToSQLUpdate(q: DBQuery<any>, record: Record<any, any>): string {
+  // var sql = mysql.format('UPDATE posts SET modified = ? WHERE id = ?', [CURRENT_TIMESTAMP, 42]);
+  const tokens = [
+    `UPDATE`,
+    mysql.escapeId(q.table),
+    `SET`,
+    Object.keys(record)
+      .map(f => mysql.escapeId(mapNameToMySQL(f)) + ' = ?')
+      .join(', '),
+    ...whereTokens(q),
+  ]
+
+  return mysql.format(tokens.join(' '), Object.values(record))
+}
+
+function selectTokens(q: DBQuery<any>): string[] {
+  let fields = ['*']
+
+  if (q._selectedFieldNames) {
+    fields = q._selectedFieldNames.length ? q._selectedFieldNames : ['id']
+  }
+
+  // We don't do `escapeId` cause it'll ruin e.g SELECT `count *` FROM ...
+  return [`SELECT`, fields.map(f => mapNameToMySQL(f)).join(', '), `FROM`, mysql.escapeId(q.table)]
+}
+
+function offsetLimitTokens(q: DBQuery<any>): string[] {
+  const tokens: string[] = []
+
+  if (q._limitValue) {
+    tokens.push(`LIMIT`, String(q._limitValue))
+
+    // In SQL OFFSET is only allowed if LIMIT is set
+    if (q._offsetValue) {
+      tokens.push(`OFFSET`, String(q._offsetValue))
+    }
+  }
+
+  return tokens
+}
+
+function orderTokens(q: DBQuery<any>): string[] {
+  if (!q._orders.length) return []
+  return [
+    `ORDER BY`,
+    q._orders.map(o => `\`${o.name}\` ${o.descending ? 'DESC' : 'ASC'}`).join(', '),
+  ]
+}
+
+const OP_MAP: Partial<Record<DBQueryFilterOperator, string>> = {
+  '==': '=',
+}
+
+function whereTokens(q: DBQuery<any>): string[] {
+  if (!q._filters.length) return []
+
+  return [
+    `WHERE`,
+    q._filters
+      .map(f => {
+        if (f.val === null || f.val === undefined) {
+          // special treatment
+
+          return [
+            mysql.escapeId(mapNameToMySQL(f.name)),
+            f.op === '==' ? 'IS NULL' : 'IS NOT NULL',
+          ].join(' ')
+        }
+
+        if (Array.isArray(f.val)) {
+          // special case for arrays
+          return `${mysql.escapeId(mapNameToMySQL(f.name))} IN (${mysql.escape(f.val)})`
+        }
+
+        return [
+          mysql.escapeId(mapNameToMySQL(f.name)),
+          OP_MAP[f.op] || f.op,
+          mysql.escape(f.val),
+        ].join(' ')
+      })
+      .join(' AND '),
+  ]
+}

package/src/schema/mysql.schema.util.ts

@@ -0,0 +1,131 @@
+import {
+  JsonSchemaBoolean,
+  JsonSchemaNumber,
+  JsonSchemaObject,
+  JsonSchemaRootObject,
+  JsonSchemaString,
+} from '@naturalcycles/js-lib'
+import * as mysql from 'mysql'
+
+export interface MySQLTableStats {
+  Field: string // created
+  Type: string // int(11)
+  Null: string // 'YES'
+}
+
+export interface MySQLSchemaOptions {
+  /**
+   * @default 'InnoDB'
+   */
+  engine?: string
+}
+
+/**
+ * It currently skips nullability and declares everything as "DEFAULT NULL".
+ */
+export function jsonSchemaToMySQLDDL(
+  table: string,
+  schema: JsonSchemaObject<any>,
+  opt: MySQLSchemaOptions = {},
+): string {
+  const { engine = 'InnoDB' } = opt
+
+  const lines: string[] = [`CREATE TABLE ${mysql.escapeId(table)} (`]
+
+  const innerLines = Object.entries(schema.properties).map(([k, s]) => {
+    if (k === 'id') {
+      return `id VARCHAR(255) NOT NULL`
+    }
+
+    let type
+
+    if (s.type === 'string') {
+      // can specify isoDate later
+      type = 'LONGTEXT'
+    } else if (s.type === 'integer') {
+      type = 'INT(11)'
+    } else if (s.type === 'number') {
+      if (['unixTimestamp', 'int32'].includes((s as JsonSchemaNumber).format!)) {
+        type = 'INT(11)'
+      } else {
+        type = 'FLOAT(11)'
+      }
+    } else if (s.type === 'boolean') {
+      type = 'TINYINT(1)'
+    } else if (s.instanceof === 'Buffer') {
+      type = 'LONGBLOB'
+    } else if (s.type === 'null') {
+      type = 'VARCHAR(255)'
+    } else if (s.type === 'array') {
+      type = 'LONGTEXT' // to be JSON.stringified?
+    } else if (s.type === 'object') {
+      type = 'LONGTEXT' // to be JSON.stringified?
+    } else {
+      // default
+      type = 'LONGTEXT'
+    }
+
+    const tokens: string[] = [mysql.escapeId(mapNameToMySQL(k)), type, `DEFAULT NULL`]
+
+    return tokens.join(' ')
+  })
+
+  innerLines.push(`PRIMARY KEY (id)`)
+
+  lines.push(innerLines.join(',\n'))
+  lines.push(`) ENGINE=${engine}`)
+
+  return lines.join('\n')
+}
+
+export function mysqlTableStatsToJsonSchemaField<T = any>(
+  table: string,
+  stats: MySQLTableStats[],
+): JsonSchemaRootObject<T> {
+  const s: JsonSchemaRootObject<T> = {
+    $id: `${table}.schema.json`,
+    type: 'object',
+    properties: {} as any,
+    required: [],
+    additionalProperties: true,
+  }
+
+  stats.forEach(stat => {
+    const name = stat.Field
+    const t = stat.Type.toLowerCase()
+    const notNull = stat.Null?.toUpperCase() !== 'YES'
+    if (notNull) {
+      s.required.push(name as any)
+    }
+
+    if (t.includes('text') || t.includes('char')) {
+      s.properties[name] = { type: 'string' } as JsonSchemaString
+    } else if (t.includes('lob')) {
+      s.properties[name] = { instanceof: 'Buffer' }
+    } else if (t.startsWith('tinyint') || t.includes('(1)')) {
+      s.properties[name] = { type: 'boolean' } as JsonSchemaBoolean
+    } else if (t.startsWith('int(')) {
+      s.properties[name] = { type: 'integer' } as JsonSchemaNumber
+    } else if (t.startsWith('float')) {
+      s.properties[name] = { type: 'number' } as JsonSchemaNumber
+    } else {
+      console.log(s)
+      throw new Error(`Unknown mysql field type ${name} ${stat.Type}`)
+    }
+  })
+
+  return s
+}
+
+/**
+ * Because MySQL doesn't support `.` in field names and escapes them as tableName + fieldName.
+ *
+ * @param name
+ */
+export function mapNameToMySQL(name: string): string {
+  return name.replace(/\./g, '_dot_')
+}
+
+export function mapNameFromMySQL(name: string): string {
+  return name.replace(/_dot_/g, '.')
+}