leoric 1.11.1 → 1.12.0

package/History.md

@@ -1,3 +1,10 @@
+1.12.0 / 2021-10-12
+===================
+
+  * feat: support custom fields query and sequelize mode export rawAttributes (#192)
+  * refactor: collection format query result (#194)
+  * refactor: object condition parsing and expression formatting (#191)
+
 1.11.1 / 2021-09-28
 ===================
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "leoric",
-  "version": "1.11.1",
+  "version": "1.12.0",
   "description": "JavaScript Object-relational mapping alchemy",
   "main": "index.js",
   "types": "types/index.d.ts",
@@ -20,6 +20,7 @@
     "test:mysql2": "./test/start.sh test/integration/mysql2.test.js",
     "test:postgres": "./test/start.sh test/integration/postgres.test.js",
     "test:sqlite": "./test/start.sh test/integration/sqlite.test.js",
+    "test:dts": "./test/start.sh test/types/dts.test.js",
     "test:coverage": "nyc ./test/start.sh && nyc report --reporter=lcov",
     "lint": "eslint ./",
     "lint:fix": "eslint . --fix"
@@ -57,6 +58,8 @@
     "@babel/eslint-parser": "^7.14.7",
     "@cara/minami": "^1.2.3",
     "@journeyapps/sqlcipher": "^5.2.0",
+    "@types/node": "^16.10.1",
+    "coffee": "^5.4.0",
     "dayjs": "^1.10.3",
     "eslint": "^7.20.0",
     "expect.js": "^0.3.1",
@@ -67,6 +70,7 @@
     "nyc": "^15.1.0",
     "pg": "^8.5.1",
     "sinon": "^10.0.0",
-    "sqlite3": "^5.0.2"
+    "sqlite3": "^5.0.2",
+    "typescript": "^4.4.3"
   }
 }

package/src/adapters/sequelize.js

@@ -130,6 +130,10 @@ module.exports = Bone => {
       return this;
     }
 
+    static get rawAttributes() {
+      return this.attributes;
+    }
+
     /**
      * add scope see https://sequelize.org/master/class/lib/model.js~Model.html#static-method-addScope
      *

package/src/bone.js

@@ -1147,19 +1147,30 @@ class Bone {
    * @return {Bone}
    */
   static instantiate(row) {
-    const { attributes, driver } = this;
+    const { attributes, driver, attributeMap } = this;
     const instance = new this();
 
-    for (const name in attributes) {
-      const { columnName, jsType } = attributes[name];
-      if (columnName in row) {
+    for (const columnName in row) {
+      const value = row[columnName];
+      if (attributeMap.hasOwnProperty(columnName)) {
+        const { jsType, name } = attributeMap[columnName];
         // to make sure raw and rawSaved hold two different objects
-        instance._setRaw(name, driver.cast(row[columnName], jsType));
-        instance._setRawSaved(name, driver.cast(row[columnName], jsType));
+        instance._setRaw(name, driver.cast(value, jsType));
+        instance._setRawSaved(name, driver.cast(value, jsType));
       } else {
+        if (!isNaN(value)) instance[columnName] = Number(value);
+        else if (!isNaN(Date.parse(value))) instance[columnName] = new Date(value);
+        else instance[columnName] = value;
+      }
+    }
+
+    for (const name in attributes) {
+      const { columnName } = attributes[name];
+      if (!(columnName in row)) {
         instance._getRawUnset().add(name);
       }
     }
+
     instance.isNewRecord = false;
     return instance;
   }

package/src/collection.js

@@ -1,7 +1,5 @@
 'use strict';
 
-const Spell = require('./spell');
-
 /**
  * An extended Array to represent collections of models.
  */
@@ -14,8 +12,7 @@ class Collection extends Array {
    * @returns {Collection|Array}
    */
   static init({ spell, rows, fields }) {
-    if (isDispatchable(spell)) return dispatch(spell, rows, fields);
-    return convert(spell, rows, fields);
+    return dispatch(spell, rows, fields);
   }
 
   /**
@@ -55,92 +52,98 @@ class Collection extends Array {
 }
 
 /**
- * Check if current spell is eligible to build models.
- * @param {Spell} spell
- * @returns {boolean}
+ * get all columns
+ * @param {Spell} spell 
+ * @returns {Array<String>} columns
  */
-function isDispatchable(spell) {
-  const { columns, groups, table } = spell;
-
-  for (const token of columns) {
-    const { type } = token;
-    if (type === 'func') return false;
-    if (type === 'alias' && token.args[0].type === 'func') return false;
-  }
-  if (groups.length > 0) return false;
-  if (table.value instanceof Spell && Object.keys(table.value.joins).length > 0) {
-    return false;
+function getColumns(spell) {
+  const { joins, columns, Model } = spell;
+  const { tableAlias, table, driver } = Model;
+  if (!columns.length) return [];
+  let targetColumns = [];
+  const joined = Object.keys(joins).length > 0;
+  if (driver.type === 'sqlite' && joined) {
+    /**
+     * { type: 'alias', value: 'posts:id', args: [ { type: 'id', qualifiers: [ 'posts' ], value: 'ss' } ] },
+     */
+    if (columns.length) {
+      for (const column of columns) {
+        targetColumns.push(...column.args.filter(
+          c => c.qualifiers && (c.qualifiers.includes(table) || c.qualifiers.includes(tableAlias))
+        ).map(c => c.value));
+      }
+    }
+  } else {
+    targetColumns = columns.filter(c => !c.qualifiers || c.qualifiers.includes(table) || c.qualifiers.includes(tableAlias)).map(v => v.value);
   }
-  return true;
+  return targetColumns;
 }
 
 /**
- * Convert the results to collection that consists of models with their associations set up according to `spell.joins`.
- * @private
- * @param {Spell} spell
- * @param {Object[]} rows
- * @param {Object[]} fields
- * @returns {Collection}
+ * join target instantiatable or not
+ * @param {Object} join
+ * @returns {boolean}
  */
-function dispatch(spell, rows, fields) {
-  const { Model } = spell;
-
-  if (Object.keys(spell.joins).length === 0) {
-    return Collection.from(rows, row => Model.instantiate(Object.values(row)[0]));
-  }
-
-  const results = new Collection();
-  const { tableAlias, table, primaryColumn, primaryKey } = Model;
-
-  for (const row of rows) {
-    // If SQL contains subqueries, such as `SELECT * FROM (SELECT * FROM foo) AS bar`,
-    // the table name of the columns in SQLite is the original table name instead of the alias.
-    // Hence we need to fallback to original table name here.
-    const main = row[tableAlias] || row[table];
-    let current = results.find(result => result[primaryKey] == main[primaryColumn]);
-
-    if (!current) {
-      current = Model.instantiate(main);
-      results.push(current);
+function joinedInstantiatable(join) {
+  const { on, Model } = join;
+  const { tableAlias, table } = Model;
+  let columns = [];
+  if (on && on.args && on.args.length) {
+    for (const arg of on.args) {
+      const { type } = arg;
+      // { type: 'op', ..., args: [{ value: '', qualifiers: [] }] }
+      if (type === 'op' && arg.args && arg.args.length) {
+        columns.push(...arg.args.filter(
+          c => c.qualifiers && (c.qualifiers.includes(table) || c.qualifiers.includes(tableAlias))
+        ).map(c => c.value));
+      } else if (arg.value && arg.qualifiers && (arg.qualifiers.includes(table) || arg.qualifiers.includes(tableAlias))) {
+        // { type: 'id', value: '', qualifiers: [] }
+        columns.push(arg.value);
+      }
     }
-
-    dispatchJoins(current, spell, row, fields);
   }
+  if (!columns.length) return true;
+  const attributeKeys = Object.keys(Model.attributes);
+  return columns.some(r => attributeKeys.includes(r));
+}
 
-  return results;
+/**
+ * @param {Spell} spell 
+ * @returns duplicate main Model
+ */
+function shouldFindJoinTarget(spell) {
+  const { Model } = spell;
+  const { primaryKey } = Model;
+  const columns = getColumns(spell);
+  return !columns.length || columns.length && columns.includes(primaryKey);
 }
 
-function dispatchJoins(current, spell, row, fields) {
-  for (const qualifier in spell.joins) {
-    const { Model, hasMany } = spell.joins[qualifier];
-    // mysql2 nests rows with table name instead of table alias.
-    const values = row[qualifier] || row[Model.table];
-    const id = values[Model.primaryColumn];
-    if (hasMany) {
-      if (!current[qualifier]) current[qualifier] = new Collection();
-      if (!id || current[qualifier].some(item => item[Model.primaryKey] === id)) continue;
-      current[qualifier].push(Model.instantiate(values));
-    } else {
-      current[qualifier] = Object.values(values).some(value => value != null)
-        ? Model.instantiate(values)
-        : null;
-    }
-  }
+/**
+ * @param {Spell} spell 
+ * @returns {boolean}
+ */
+function instantiatable(spell) {
+  const { Model, groups } = spell;
+  const { attributes } = Model;
+  const columns = getColumns(spell);
+  if (groups.length > 0) return false;
+  if (!columns.length) return true;
+  const attributeKeys = Object.keys(attributes);
+  return columns.some(r => attributeKeys.includes(r));
 }
 
+
 /**
- * Convert returned rows to result set by translating columns (if found) to attributes.
+ * Convert the results to collection that consists of models with their associations set up according to `spell.joins`.
  * @private
  * @param {Spell} spell
  * @param {Object[]} rows
  * @param {Object[]} fields
- * @returns {Object[]}
+ * @returns {Collection}
  */
-function convert(spell, rows, fields) {
-  const results = [];
-  const { groups, joins, columns } = spell;
-  const { table, tableAlias } = spell.Model;
-
+function dispatch(spell, rows, fields) {
+  const { groups, joins, columns, Model } = spell;
+  const { tableAlias, table, primaryKey, primaryColumn, attributes } = Model;
   // await Post.count()
   if (rows.length <= 1 && columns.length === 1 && groups.length === 0) {
     const { type, value, args } = columns[0];
@@ -151,40 +154,69 @@ function convert(spell, rows, fields) {
     }
   }
 
+  const joined = Object.keys(joins).length > 0;
+  const shouldFindDuplicate = shouldFindJoinTarget(spell);
+  const canInstantiate = instantiatable(spell);
+  const attributeKeys = Object.keys(attributes);
+
+  const results = new Collection();
   for (const row of rows) {
-    const result = { '': {} };
-    for (let prop in row) {
+    const result = {};
+    for (const prop in row) {
       const data = row[prop];
-      // mysql2 sometimes nests rows with table name instead of table alias
       const qualifier = prop === table ? tableAlias : prop;
-      const obj = result[qualifier] || (result[qualifier] = {});
-      if (qualifier === '') {
-        Object.assign(obj, data);
+      if (qualifier === '' || qualifier === tableAlias) {
+        Object.assign(result, data);
+      } else {
+        if (Object.values(data).some(value => value != null)) result[prop] = data;
       }
-      else if (qualifier in joins || qualifier == tableAlias) {
-        const { Model } = joins[qualifier] || spell;
-        for (const columnName in data) {
-          const definition = Model.attributeMap[columnName];
-          if (definition) {
-            obj[definition.name] = data[columnName];
-          } else {
-            result[''][columnName] = data[columnName];
+    }
+    let current;
+    if (shouldFindDuplicate && result[primaryColumn] != null) {
+      current = results.find(r => r[primaryKey] == result[primaryColumn]);
+    }
+    if (!current) {
+      const resultKeys = Object.keys(result);
+      current = canInstantiate || (!groups.length && resultKeys.some(c => attributeKeys.includes(c)))? Model.instantiate(result) : result;
+      results.push(current);
+    }
+    if (joined) {
+      dispatchJoins(current, spell, row, fields);
+    }
+  }
+
+  return results;
+}
+
+function dispatchJoins(current, spell, row, fields) {
+  const instantiatableMap = {};
+  for (const qualifier in spell.joins) {
+    const join = spell.joins[qualifier];
+    const { Model, hasMany } = join;
+    if (instantiatableMap[qualifier] === undefined) instantiatableMap[qualifier] = joinedInstantiatable(join);
+    const joinInstantiatable = instantiatableMap[qualifier];
+    // mysql2 nests rows with table name instead of table alias.
+    const values = row[qualifier] || row[Model.table];
+    if (values) {
+      if (hasMany) {
+        const id = values[Model.primaryColumn];
+        if (!current[qualifier]) current[qualifier] = new Collection();
+        if (!Array.isArray(current[qualifier])) {
+          const origin = !(current[qualifier] instanceof Model) && joinInstantiatable? Model.instantiate(current[qualifier]) : current[qualifier];
+          current[qualifier] = new Collection();
+          if (Object.values(values).some(value => value != null)) {
+            current[qualifier].push(origin);
           }
         }
-      }
-      else {
-        throw new Error(`Unknown qualifier ${qualifier}`);
+        if (!id || current[qualifier].some(item => item[Model.primaryKey] === id) || Object.values(values).every(value => value == null)) continue;
+        current[qualifier].push(joinInstantiatable? Model.instantiate(values) : values);
+      } else {
+        current[qualifier] = Object.values(values).some(value => value != null)
+          ? (joinInstantiatable? Model.instantiate(values) : values)
+          : null;
       }
     }
-    results.push(result);
   }
-
-  if (Object.keys(joins).length > 0) return results;
-  return results.map(result => {
-    const merged = {};
-    for (const obj of Object.values(result)) Object.assign(merged, obj);
-    return merged;
-  });
 }
 
 module.exports = Collection;

package/src/drivers/abstract/spellbook.js

@@ -2,25 +2,8 @@
 
 const SqlString = require('sqlstring');
 
-const { precedes, copyExpr, findExpr, walkExpr } = require('../../expr');
-
-/**
- * Find model by qualifiers.
- * @example
- * findModel(spell, ['comments'])
- * findModel(spell)
- *
- * @param {Spell} spell
- * @param {string[]} qualifiers
- */
-function findModel(spell, qualifiers) {
-  const qualifier = qualifiers && qualifiers[0];
-  const Model = qualifier && qualifier != spell.Model.tableAlias
-    ? (spell.joins.hasOwnProperty(qualifier) ? spell.joins[qualifier].Model : null)
-    : spell.Model;
-  if (!Model) throw new Error(`Unabled to find model ${qualifiers}`);
-  return Model;
-}
+const { copyExpr, findExpr, walkExpr } = require('../../expr');
+const { formatExpr, formatConditions, collectLiteral } = require('../../expr_formatter');
 
 /**
  * Format orders into ORDER BY clause in SQL
@@ -29,185 +12,11 @@ function findModel(spell, qualifiers) {
  */
 function formatOrders(spell, orders) {
   return orders.map(([token, order]) => {
-    const column = formatColumn(spell, token);
+    const column = formatExpr(spell, token);
     return order == 'desc' ? `${column} DESC` : column;
   });
 }
 
-/**
- * Format token into identifiers/functions/etc. in SQL
- * @example
- * formatColumn(spell, { type: 'id', value: 'title' })
- * // => `title`
- *
- * formatColumn(spell, { type: 'func', name: 'year', args: [ { type: 'id', value: 'createdAt' } ] })
- * // => YEAR(`createdAt`)
- *
- * @param {Spell}  spell
- * @param {Object} token
- */
-function formatColumn(spell, token) {
-  if (token.type == 'id') {
-    return formatIdentifier(spell, token);
-  } else {
-    return formatExpr(spell, token);
-  }
-}
-
-/**
- * Format identifiers into escaped string with qualifiers.
- * @param {Spell}  spell
- * @param {Object} ast
- */
-function formatIdentifier(spell, ast) {
-  const { value, qualifiers } = ast;
-  const Model = findModel(spell, qualifiers);
-  const column = Model.unalias(value);
-  const { escapeId } = spell.Model.driver;
-
-  if (qualifiers && qualifiers.length > 0) {
-    return `${qualifiers.map(escapeId).join('.')}.${escapeId(column)}`;
-  } else {
-    return escapeId(column);
-  }
-}
-
-const extractFieldNames = ['year', 'month', 'day'];
-
-function formatFuncExpr(spell, ast) {
-  const { name, args } = ast;
-  const { type } = spell.Model.driver;
-
-  // https://www.postgresql.org/docs/9.1/static/functions-datetime.html
-  if (type === 'postgres' && extractFieldNames.includes(name)) {
-    return `EXTRACT(${name.toUpperCase()} FROM ${args.map(arg => formatExpr(spell, arg)).join(', ')})`;
-  }
-
-  return `${name.toUpperCase()}(${args.map(arg => formatExpr(spell, arg)).join(', ')})`;
-}
-
-/**
- * The `... IS NULL` predicate is not parameterizable.
- * - https://github.com/brianc/node-postgres/issues/1751
- * @param {Array} values the collected values
- * @param {Object} ast the abstract syntax tree
- * @returns {Array} values
- */
-function collectLiteral(values, ast) {
-  walkExpr(ast, ({ type, value }) => {
-    if (type == 'literal' && value != null) {
-      if (Array.isArray(value)) {
-        values.push(...value);
-      } else {
-        values.push(value);
-      }
-    }
-  });
-  return values;
-}
-
-function formatLiteral(spell, ast) {
-  const { value } = ast;
-
-  if (value == null) {
-    return 'NULL';
-  } else if (Array.isArray(value)) {
-    if (value.length) return `(${value.map(() => '?').join(', ')})`;
-    return '(NULL)';
-  } else {
-    return '?';
-  }
-}
-
-/**
- * Format the abstract syntax tree of an expression into escaped string.
- * @param {Spell}  spell
- * @param {Object} ast
- */
-function formatExpr(spell, ast) {
-  const { type, name, value, args } = ast;
-  switch (type) {
-    case 'literal':
-      return formatLiteral(spell, ast);
-    case 'subquery':
-      return `(${value.toSqlString()})`;
-    case 'wildcard':
-      return '*';
-    case 'alias':
-      return `${formatExpr(spell, args[0])} AS ${formatIdentifier(spell, ast)}`;
-    case 'mod':
-      return `${name.to.toUpperCase()} ${formatExpr(spell, args[0])}`;
-    case 'id':
-      return formatIdentifier(spell, ast);
-    case 'op':
-      return formatOpExpr(spell, ast);
-    case 'func':
-      return formatFuncExpr(spell, ast);
-    case 'raw':
-      // return value directly
-      return value;
-    default:
-      throw new Error(`Unexpected type ${type}`);
-  }
-}
-
-/**
- * Check if current token is logical operator or not, e.g. `AND`/`NOT`/`OR`.
- * @param {Object} ast
- */
-function isLogicalOp({ type, name }) {
-  return type == 'op' && ['and', 'not', 'or'].includes(name);
-}
-
-/**
- * Format `{ type: 'op' }` expressions into escaped string.
- * @param {Spell}  spell
- * @param {Object} ast
- */
-function formatOpExpr(spell, ast) {
-  const { name, args } = ast;
-  const params = args.map(arg => {
-    return isLogicalOp(ast) && isLogicalOp(arg) && precedes(name, arg.name) < 0
-      ? `(${formatExpr(spell, arg)})`
-      : formatExpr(spell, arg);
-  });
-
-  if (name == 'between' || name == 'not between') {
-    return `${params[0]} ${name.toUpperCase()} ${params[1]} AND ${params[2]}`;
-  }
-  else if (name == 'not') {
-    return `NOT ${params[0]}`;
-  }
-  else if ('!~-'.includes(name) && params.length == 1) {
-    return `${name} ${params[0]}`;
-  }
-  else if (args[1].type == 'literal' && args[1].value == null && !isLogicalOp(ast)) {
-    if (['=', '!='].includes(name)) {
-      const op = name == '=' ? 'IS' : 'IS NOT';
-      return `${params[0]} ${op} NULL`;
-    } else {
-      throw new Error(`Invalid operator ${name} against null`);
-    }
-  }
-  // IN (1, 2, 3)
-  // IN (SELECT user_id FROM group_users)
-  else if ((args[1].type == 'literal' && Array.isArray(args[1].value)) || args[1].type == 'subquery') {
-    let op = name;
-    if (name == '=') {
-      op = 'in';
-    } else if (name == '!=') {
-      op = 'not in';
-    }
-    if (['in', 'not in'].includes(op)) {
-      return `${params[0]} ${op.toUpperCase()} ${params[1]}`;
-    } else {
-      throw new Error(`Invalid operator ${name} against ${args[1].value}`);
-    }
-  } else if (params[1] !== '') {
-    return `${params[0]} ${name.toUpperCase()} ${params[1]}`;
-  }
-}
-
 /**
  * Format a spell without joins into a full SELECT query. This function is also used to format the subquery which is then used as a drived table in a SELECT with joins.
  * @param {Spell} spell
@@ -228,7 +37,7 @@ function formatSelectWithoutJoin(spell) {
     columns.reduce(collectLiteral, values);
     const selects = [];
     for (const token of columns) {
-      const column = formatColumn(spell, token);
+      const column = formatExpr(spell, token);
       if (!selects.includes(column)) selects.push(column);
     }
     chunks.push(`${selects.join(', ')}`);
@@ -254,7 +63,7 @@ function formatSelectWithoutJoin(spell) {
   }
 
   if (groups.length > 0) {
-    const groupColumns = groups.map(group => formatColumn(spell, group));
+    const groupColumns = groups.map(group => formatExpr(spell, group));
     chunks.push(`GROUP BY ${groupColumns.join(', ')}`);
   }
 
@@ -371,7 +180,7 @@ function formatSelectExpr(spell, values) {
 
   for (const token of columns) {
     collectLiteral(values, token);
-    const selectExpr = formatColumn(spell, token);
+    const selectExpr = formatExpr(spell, token);
     const qualifier = token.qualifiers ? token.qualifiers[0] : '';
     const list = map[qualifier] || (map[qualifier] = []);
     list.push(selectExpr);
@@ -444,7 +253,7 @@ function formatSelectWithJoin(spell) {
   }
 
   if (groups.length > 0) {
-    chunks.push(`GROUP BY ${groups.map(group => formatColumn(spell, group)).join(', ')}`);
+    chunks.push(`GROUP BY ${groups.map(group => formatExpr(spell, group)).join(', ')}`);
   }
 
   if (havingConditions.length > 0) {
@@ -513,22 +322,6 @@ function formatDelete(spell) {
   }
 }
 
-/**
- * Format an array of conditions into an expression. Conditions will be joined with `AND`.
- * @param {Object[]} conditions - An array of parsed where/having/on conditions
- */
-function formatConditions(spell, conditions) {
-  return conditions
-    .map(condition => {
-      return isLogicalOp(condition) && condition.name == 'or' && conditions.length > 1
-        ? `(${formatExpr(spell, condition)})`
-        : formatExpr(spell, condition);
-    })
-    // filter empty condition
-    .filter((condition) => !!condition)
-    .join(' AND ');
-}
-
 /**
  * Format a spell into INSERT query.
  * @param {Spell} spell