@igojs/db 6.0.0-beta.1

package/index.js

@@ -0,0 +1,27 @@
+
+const context = require('./src/context');
+
+// Initialize @igojs/db with dependencies from @igojs/server
+function init({ config, cache, logger, utils, errorhandler }) {
+  context.config = config;
+  context.cache = cache;
+  context.logger = logger;
+  context.utils = utils;
+  context.errorhandler = errorhandler;
+}
+
+module.exports = {
+  init,
+  get Model() { return require('./src/Model'); },
+  get Query() { return require('./src/Query'); },
+  get CachedQuery() { return require('./src/CachedQuery'); },
+  get Schema() { return require('./src/Schema'); },
+  get Sql() { return require('./src/Sql'); },
+  get Db() { return require('./src/Db'); },
+  get dbs() { return require('./src/dbs'); },
+  get migrations() { return require('./src/migrations'); },
+  get DataTypes() { return require('./src/DataTypes'); },
+  get CacheStats() { return require('./src/CacheStats'); },
+  get PaginatedOptimizedQuery() { return require('./src/PaginatedOptimizedQuery'); },
+  get PaginatedOptimizedSql() { return require('./src/PaginatedOptimizedSql'); },
+};

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@igojs/db",
+  "version": "6.0.0-beta.1",
+  "description": "Igo ORM - Database abstraction layer for MySQL and PostgreSQL",
+  "main": "index.js",
+  "scripts": {
+    "test": "mocha --exit 'test/**/*.js'"
+  },
+  "keywords": [
+    "orm",
+    "mysql",
+    "postgresql",
+    "database"
+  ],
+  "author": "@igocreate",
+  "license": "ISC",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "lodash": "^4.17.21"
+  },
+  "optionalDependencies": {
+    "mysql2": "^3.15.3",
+    "pg": "^8.16.3"
+  }
+}

package/src/CacheStats.js

@@ -0,0 +1,33 @@
+const _         = require('lodash');
+const context   = require('./context');
+
+const NAMESPACE = '_cache_statistics';
+
+//
+module.exports.incr = (key, type) => {
+  const { cache } = context;
+  cache.incr(NAMESPACE, `${key}.${type}`);
+};
+
+//
+module.exports.getStats = async () => {
+  const { cache } = context;
+  const statistics = {};
+
+  await cache.scan(`${NAMESPACE}/*`, async (key) => {
+    key = key.substr(NAMESPACE.length + 1);
+    const value = await cache.get(NAMESPACE, key);
+    _.set(statistics, key, value);
+  });
+
+  _.each(statistics, (statistic, key) => {
+    let { hits, misses } = statistic;
+    hits            = hits || 0;
+    misses          = misses || 0;
+    statistic.table = key;
+    statistic.total = hits + misses;
+    statistic.rate  = Math.round(hits / statistic.total * 100);
+  });
+
+  return _.values(statistics);
+};

package/src/CachedQuery.js

@@ -0,0 +1,40 @@
+
+const context     = require('./context');
+
+const Query       = require('./Query');
+const CacheStats  = require('./CacheStats');
+
+//
+module.exports = class CachedQuery extends Query {
+
+  async runQuery() {
+    const { cache } = context;
+    const { query, schema } = this;
+    const sqlQuery  = this.toSQL();
+    const db        = this.getDb();
+
+    const namespace = '_cached.' + query.table;
+
+    if (query.verb !== 'select') {
+      cache.flush(namespace + '/*'); // non-blocking flush for performance
+      const result = await db.query(sqlQuery.sql, sqlQuery.params, query.options);
+      return result;
+    }
+
+    const key = JSON.stringify(sqlQuery);
+    let type  = 'hits';
+
+    const result = await cache.fetch(
+      namespace,
+      key,
+      async () => {
+        type = 'misses';
+        return await db.query(sqlQuery.sql, sqlQuery.params, query.options);
+      },
+      schema.cache.ttl
+    );
+
+    CacheStats.incr(query.table, type);
+    return result;
+  }
+};

package/src/DataTypes.js

@@ -0,0 +1,23 @@
+
+const _       = require('lodash');
+const context = require('./context');
+
+
+module.exports = {
+  default: {
+    set:  _.identity,
+    get:  _.identity,
+  },
+  boolean: {
+    set:  value => (value === null || value === undefined) ? null : !!value,
+    get:  value => (value === null || value === undefined) ? null : !!value,
+  },
+  json: {
+    set:  value => context.utils.toJSON(value),
+    get:  value => context.utils.fromJSON(value),
+  },
+  array: {
+    set:  value => value && Array.isArray(value) ? value.join(',') : value,
+    get:  value => value && value.split ? value.split(',') : []
+  },
+};

package/src/Db.js

@@ -0,0 +1,147 @@
+
+const _    = require('lodash');
+const context = require('./context');
+
+// Dynamic driver loading
+let loadedDriver = null;
+const getDriver = (driverName) => {
+  if (!loadedDriver) {
+    loadedDriver = require(`./drivers/${driverName}`);
+  }
+  return loadedDriver;
+};
+
+//
+const logQuery = (sql, params, err) => {
+  const { logger, errorhandler } = context;
+  const _log = err ? logger.error : logger.info;
+  _log('Db.query: ' + sql);
+  if (params?.length) {
+    _log('With params: ' + params);
+  }
+  if (err) {
+    errorhandler.errorSQL(err);
+  }
+}
+
+
+//
+class Db {
+
+  constructor(name) {
+    const { config } = context;
+    this.name       = name;
+    this.config     = config[name];
+    this.driver     = getDriver(this.config.driver);
+    this.connection = null;
+    this.config.migrations_dir = `sql/${this.name}`;
+  }
+
+  async init() {
+    const { config } = context;
+    this.pool       = await this.driver.createPool(this.config);
+    this.connection = null;
+    this.TEST_ENV   = config.env === 'test';
+  }
+
+  //
+  async getConnection() {
+    const { driver, pool, TEST_ENV } = this;
+    // if connection is in local storage
+    if (TEST_ENV && this.connection) {
+      // console.log('keep same connection');
+      return { connection: this.connection, keep: true };
+    }
+
+    const connection = await driver.getConnection(pool);
+
+    if (TEST_ENV) {
+      this.connection = connection;
+    }
+    return { connection, keep: false };
+
+  }
+
+  //
+  async query(sql, params=[], options={}) {
+    const { logger } = context;
+    const { driver, config, TEST_ENV } = this;
+    const { dialect } = driver;
+
+    const runquery = async() => {
+
+      const { connection, keep } = await this.getConnection();
+
+      try {
+
+        const result = await this.driver.query(connection, sql, params, options);
+        if (config.debugsql) {
+          logQuery(sql, params);
+        }
+        return dialect.getRows(result);
+
+      } catch (err) {
+        if (options.silent) {
+          return;
+        }
+        // log & rethrow error
+        logQuery(sql, params, err);
+        throw err;
+
+      } finally {
+        if (!keep) {
+          // console.log('query: release transaction');
+          driver.release(connection);
+          if (TEST_ENV) {
+            this.connection = null;
+          }
+        }
+      }
+    };
+
+    if (this.pool) {
+      return await runquery();
+    }
+
+    logger.info('Db.query: Trying to reinitialize db connection pool');
+    await this.init();
+    if (!this.pool) {
+      logger.error('could not create db connection pool');
+    } else {
+      return await runquery();
+    }
+  }
+
+
+
+  //
+  async beginTransaction() {
+    const { driver } = this;
+    const { connection } = await this.getConnection();
+    await driver.beginTransaction(connection);
+  }
+
+  //
+  async commitTransaction() {
+    const { driver, TEST_ENV } = this;
+    const { connection } = await this.getConnection();
+    await driver.commit(connection);
+    driver.release(connection);
+    if (TEST_ENV) {
+      this.connection = null;
+    }
+  }
+
+  //
+  async rollbackTransaction() {
+    const { driver, TEST_ENV } = this;
+    const { connection } = await this.getConnection();
+    await driver.rollback(connection);
+    driver.release(connection);
+    if (TEST_ENV) {
+      this.connection = null;
+    }
+  }
+}
+
+module.exports = Db;

package/src/Model.js

@@ -0,0 +1,261 @@
+const _         = require('lodash');
+const CachedQuery = require('./CachedQuery');
+
+const Query     = require('./Query');
+const PaginatedOptimizedQuery = require('./PaginatedOptimizedQuery');
+const Schema    = require('./Schema');
+const context   = require('./context');
+
+
+const newQuery = (constructor, verb) => {
+  if (constructor.schema.cache) {
+    return new CachedQuery(constructor, verb);
+  }
+  return new Query(constructor, verb);
+};
+
+// Simple mixin implementation to set the schema as a static attribute
+module.exports = function(schema) {
+
+  class Model {
+
+    constructor(values) {
+      _.assign(this, values);
+    }
+
+    //
+    assignValues(values) {
+      const keys = _.keys(Model.schema.colsByAttr);
+      _.assign(this, _.pick(values, keys));
+    }
+
+    // returns object with primary keys
+    primaryObject() {
+      return _.pick(this, this.constructor.schema.primary);
+    }
+
+    // update
+    async update(values) {
+      values.updated_at = new Date();
+      await this.beforeUpdate(values);
+
+      await newQuery(this.constructor, 'update')
+      .unscoped()
+      .values(values)
+      .where(this.primaryObject())
+      .execute();
+
+      if (this.constructor.schema.cache) {
+        const { cache } = context;
+        await cache.del('_cached.' + this.constructor.schema.table);
+      }
+
+    this.assignValues(values);
+    return this;
+    }
+
+    // reload
+    async reload(includes) {
+      const query = this.constructor.unscoped();
+      includes && query.includes(includes);
+      return await query.find(this.id);
+    }
+
+    // delete
+    delete() {
+      return newQuery(this.constructor, 'delete').unscoped().where(this.primaryObject()).execute();
+    }
+
+    async beforeCreate() { }
+    async beforeUpdate(values) { }
+
+
+    // find by id
+    static async find(id) {
+      return await newQuery(this).find(id);
+    }
+
+    // create
+    static async create(values, options) {
+      const _this = this;
+
+      const now = new Date();
+      const obj = new this(values);
+
+      if (this.schema.subclasses && !obj[this.schema.subclass_column]) {
+        obj[this.schema.subclass_column] = _.findKey(this.schema.subclasses, { name: this.name });
+      }
+
+      obj.created_at = obj.created_at || now;
+      obj.updated_at = obj.updated_at || now;
+
+      const create = async () => {
+        await obj.beforeCreate();
+
+        const query = newQuery(_this, 'insert').values(obj).options(options);
+        const result = await query.execute();
+
+        if (result.err) {
+          throw result.err;
+        }
+
+        const { insertId } = result;
+        if (insertId) {
+          return _this.unscoped().find(insertId);
+        }
+        return _this.unscoped().find(obj.primaryObject());
+      };
+
+      return await create();
+    }
+
+
+    // return first
+    static first() {
+      return newQuery(this).first();
+    }
+
+    // return last
+    static last() {
+      return newQuery(this).last();
+    }
+
+    // return list
+    static list() {
+      return newQuery(this).list();
+    }
+
+    //
+    static select(select) {
+      return newQuery(this).select(select);
+    }
+
+    // filter
+    static where(where, params) {
+      return newQuery(this).where(where, params);
+    }
+
+    // filter
+    static whereNot(whereNot) {
+      return newQuery(this).whereNot(whereNot);
+    }
+
+    // limit
+    static limit(limit) {
+      return newQuery(this).limit(limit);
+    }
+
+    // offset
+    static offset(offset) {
+      return newQuery(this).offset(offset);
+    }
+
+    // page
+    static page(page, nb) {
+      return newQuery(this).page(page, nb);
+    }
+
+    // order
+    static order(order) {
+      return newQuery(this).order(order);
+    }
+
+    // distinct
+    static distinct(columns) {
+      return newQuery(this).distinct(columns);
+    }
+
+    // group
+    static group(columns) {
+      return newQuery(this).group(columns);
+    }
+
+    // count
+    static count() {
+      return newQuery(this).count();
+    }
+
+    // delete
+    static delete(id, ) {
+      return newQuery(this, 'delete').unscoped().where({ id: id }).execute();
+    }
+
+    // delete all
+    static async deleteAll() {
+      return newQuery(this, 'delete').unscoped().execute();
+    }
+
+    // destroy all
+    static destroyAll() {
+      return this.deleteAll();
+    }
+
+    //
+    static update(values, ) {
+      values.updated_at = new Date();
+      return newQuery(this).unscoped().update(values, );
+    }
+
+    // includes
+    static includes(includes) {
+      return newQuery(this).includes(includes);
+    }
+
+    // includes
+    static join(associationName, columns, type='LEFT') {
+      const query = newQuery(this);
+      if (_.isString(associationName)) {
+        return query.joinOne(associationName, columns, type);
+      } else if (_.isArray(associationName)) {
+        return query.joinMany(associationName, columns, type);
+      } else if (_.isObject(associationName)) {
+        return query.joinNested(associationName);
+      }
+      console.warn('Invalid join argument for Model.join(). Must be a string, array, or object.');
+    }
+
+    //unscoped
+    static unscoped() {
+      return newQuery(this).unscoped();
+    }
+
+    //scope
+    static scope(scope) {
+      return newQuery(this).scope(scope);
+    }
+
+    /**
+     * paginatedOptimized - Retourne une PaginatedOptimizedQuery pour des requêtes optimisées avec pattern COUNT/IDS/FULL
+     *
+     * Cette méthode permet d'utiliser le pattern d'optimisation pour les requêtes avec de nombreuses jointures.
+     * Au lieu de faire un LEFT JOIN complet, on utilise :
+     * 1. COUNT avec EXISTS pour compter sans jointures
+     * 2. SELECT IDS pour récupérer les IDs avec filtres/tris/pagination
+     * 3. SELECT FULL pour récupérer les données complètes avec LEFT JOIN uniquement sur les IDs trouvés
+     *
+     * @returns {PaginatedOptimizedQuery} Instance de PaginatedOptimizedQuery
+     *
+     * Exemple d'utilisation :
+     *
+     * const result = await Folder.paginatedOptimized()
+     *   .where({ type: ['agp', 'avt'] })
+     *   .filterJoin('applicant', { last_name: 'Dupont%' })  // Filtre via EXISTS
+     *   .join('pme_folder')                                  // LEFT JOIN dans phase FULL
+     *   .order('folders.created_at DESC')
+     *   .page(1, 50);
+     *
+     * Performance :
+     * - Pour des tables de millions de lignes avec 10 jointures : amélioration de 1000x à 10000x
+     * - Le COUNT passe de plusieurs secondes à quelques millisecondes
+     * - Le SELECT bénéficie d'une pagination efficace sur les IDs seulement
+     */
+    static paginatedOptimized() {
+      return new PaginatedOptimizedQuery(this);
+    }
+
+  }
+
+  Model.schema = new Schema(schema);
+
+  return Model;
+};