mysql-warmup 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ef5600e2b1d9462f2e8653caa0ae64a1e373ae4d
+  data.tar.gz: aa4a0595091db470b66d079784d4dacadded57ca
+SHA512:
+  metadata.gz: ac489f240575c9f49f7e8bc4cb4217e899c29c2282eafffbfe1ad445085f03d806055e632ff204026acfaafdcbff2ebd33ac77e7949c7b99674df434ce789278
+  data.tar.gz: 9fcb4721c49c1a29a34e1e7349f4e066f4699a00b338041d924fb7dd9ec48ba5270f4baf25727344a2f8a5186300ae4ee39b9affe67e59552ca1c37cf6414873

data/README.md

@@ -0,0 +1,44 @@
+# mysql_warmup
+Simple mysql-wamup command tool for warming up mysql server after create/reboot
+[Github](https://github.com/manhdaovan/mysql_warmup)
+
+# Background
+With InnoDB storage engine, when you've just created new slave instance,
+first requests to DB will be hit on disk instead of buffer poll. So, the requests will be slow down.
+You can use this tool for warming up buffer poll before the first requests come.
+[Detail](https://www.percona.com/blog/2008/05/01/quickly-preloading-innodb-tables-in-the-buffer-pool/)
+
+# Usage
+1. **When?**
+  * You've set up new slave instance (base on Master-Slave model)
+  * Your mysql version < 5.6, that not support [Saving and Restoring the Buffer Pool State](https://dev.mysql.com/doc/refman/5.6/en/innodb-preload-buffer-pool.html), and your mysql instance just reboot
+  * Your mysql version >= 5.6, but you not config [Saving and Restoring the Buffer Pool State](https://dev.mysql.com/doc/refman/5.6/en/innodb-preload-buffer-pool.html), and your mysql just reboot
+2. **How?**
+  * Install this tool (please see `Install` section below)
+  * Usage as below syntax
+
+```
+    Usage: mysql-warmup -h host-or-ip -u username <options>
+
+    Input options:
+    -h host      : Host or ip of mysql instance
+    -u username  : Username to access mysql instance
+    -d database  : Database to warmup.
+                   Default to all databases exclude information_schema, mysql, performance_schema, sys
+    -p port      : Port to connect. Default to 3306
+    --help       : Show help message
+    --version    : Show mysql-warmup version
+```
+
+# Install
+* Directly with command: `$gem install mysql-warmup`
+
+* Or with `Gemfile` by adding line '`gem 'mysql-warmup`' to your `Gemfile` then `$bundle install`
+
+# TODO
+* RDoc
+
+# Development
+* All PR are welcome.
+* Be sure all test cases are passed by command: `$rake test`
+

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc 'Run tests for gmo_payment'
+task default: :test

data/bin/console

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+# -*- encoding : utf-8 -*-
+$LOAD_PATH.unshift(File.expand_path('./../lib', File.dirname(__FILE__)))
+# noinspection RubyResolve
+require 'mysql_warmup'
+require 'irb'
+require 'pp'
+
+if __FILE__ == $PROGRAM_NAME
+  IRB.start(__FILE__)
+else # check -e option
+  if /\A-e\z/ =~ $PROGRAM_NAME
+    IRB.start(__FILE__)
+  else
+    IRB.setup(__FILE__)
+  end
+end

data/bin/mysql-warmup

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+$LOAD_PATH.unshift(File.expand_path('../lib', File.dirname(__FILE__)))
+require 'mysql_warmup'
+require 'optparse'
+
+def print_help
+  puts "mysql-warmup, by ManhDV - version #{MysqlWarmup::VERSION}"
+  puts 'Website: https://github.com/manhdaovan/mysql_warmup'
+  puts
+  puts 'Usage: mysql-warmup -h host-or-ip -u username <options>'
+  puts
+  puts 'Input options:'
+  puts '-h host      : Host or ip of mysql instance'
+  puts '-u username  : Username to access mysql instance'
+  puts '-d database  : Database to warmup.'
+  puts '               Default to all databases exclude information_schema, mysql, performance_schema, sys'
+  puts '-p port      : Port to connect. Default to 3306'
+  puts '--help       : Show help message'
+  puts '--version    : Show mysql-warmup version'
+end
+
+def print_error(msg)
+  puts
+  puts ">>> ERROR: #{msg}"
+end
+
+def params_valid?(options)
+  required_params = [:host, :username]
+  valid           = true
+  required_params.each do |rp|
+    next unless options.fetch(rp, nil).nil?
+    print_error("Required #{rp}")
+    valid = false
+  end
+  valid
+end
+
+options = { show_version: false, help: false, port: 3306, database: 'all' }
+
+begin
+  OptionParser.new do |opts|
+    opts.banner = 'Usage: mysql-warmup -h host-or-ip -u username <options>'
+    opts.on('-h host', String) do |h|
+      options[:host] = h
+    end
+
+    opts.on('-u username') do |u|
+      options[:username] = u
+    end
+
+    opts.on('-d database') do |d|
+      options[:database] = d
+    end
+
+    opts.on('-p port') do |p|
+      options[:port] = p
+    end
+
+    opts.on('--help') do |_|
+      options[:help] = true
+    end
+
+    opts.on('--version') do |_|
+      options[:show_version] = true
+    end
+  end.parse!
+rescue OptionParser::InvalidArgument, OptionParser::InvalidOption => e
+  print_error(e.message)
+  print_help
+  exit(0)
+end
+
+# Execute command base on input
+if options[:show_version]
+  puts MysqlWarmup::VERSION
+elsif options[:help]
+  print_help
+elsif params_valid?(options)
+  puts
+  puts 'Input the mysql password:'
+  password = gets.chomp
+  begin
+    MysqlWarmup::Warmer.new(options[:host],
+                            options[:username],
+                            password,
+                            options[:port].to_i,
+                            options[:database]).warmup
+  rescue => e
+    print_error(e.message)
+  end
+else
+  print_help
+end

data/lib/mysql_warmup/index.rb

@@ -0,0 +1,51 @@
+module MysqlWarmup
+  class Index
+    QUERY_TEMPLATE = 'select %s from %s where %s'.freeze
+    attr_reader :query_string
+
+    def initialize(table_name, column_name, column_type, column_key)
+      @table_name   = table_name
+      @column_name  = column_name
+      @column_type  = column_type
+      @column_key   = column_key
+      @query_string = build_query_string(@table_name, @column_name,
+                                         @column_type, @column_key)
+    end
+
+    private
+
+    def build_query_string(table_name, column_name, column_type, column_key)
+      if type_primary?(column_key) && type_integer?(column_type)
+        format_query("sum(`#{table_name}`.`#{column_name}`)", "`#{table_name}`", '1')
+      elsif type_integer?(column_type)
+        format_query('count(*)', "`#{table_name}`", "`#{table_name}`.`#{column_name}` LIKE '%0%'")
+      elsif type_var_char?(column_type)
+        format_query('count(*)', "`#{table_name}`", "`#{table_name}`.`#{column_name}` LIKE '%0%'")
+      elsif type_blob?(column_type)
+        format_query('count(*)', "`#{table_name}`", "`#{table_name}`.`#{column_name}` LIKE '%0%'")
+      else
+        format_query('count(*)', "`#{table_name}`", "`#{table_name}`.`#{column_name}` LIKE '%0%'")
+      end
+    end
+
+    def type_primary?(column_key)
+      column_key.casecmp('PRI').zero?
+    end
+
+    def type_integer?(column_type)
+      !(column_type.downcase =~ /int/).nil?
+    end
+
+    def type_blob?(column_type)
+      !(column_type.downcase =~ /blob/).nil?
+    end
+
+    def type_var_char?(column_type)
+      !(column_type.downcase =~ /varchar|index/).nil?
+    end
+
+    def format_query(*params)
+      format(QUERY_TEMPLATE, *params)
+    end
+  end
+end

data/lib/mysql_warmup/logger.rb

@@ -0,0 +1,9 @@
+module MysqlWarmup
+  class Logger
+    class << self
+      def write(log_msg)
+        puts "#{Time.now}: --- #{log_msg}"
+      end
+    end
+  end
+end

data/lib/mysql_warmup/table.rb

@@ -0,0 +1,34 @@
+module MysqlWarmup
+  class Table
+    DESC_TABLE_STRUCTURE = {
+      field:   0,
+      type:    1,
+      null:    2,
+      key:     3,
+      default: 4,
+      extra:   5
+    }.freeze
+
+    attr_reader :indexes
+
+    def initialize(table_name, field_infos)
+      @table_name  = table_name
+      @field_infos = field_infos
+      @indexes     = build_index
+    end
+
+    private
+
+    def build_index
+      indexes_infos = @field_infos.select { |v| !v[DESC_TABLE_STRUCTURE[:key]].empty? }
+      indexes       = []
+      indexes_infos.each do |index_info|
+        indexes << MysqlWarmup::Index.new(@table_name,
+                                          index_info[DESC_TABLE_STRUCTURE[:field]],
+                                          index_info[DESC_TABLE_STRUCTURE[:type]],
+                                          index_info[DESC_TABLE_STRUCTURE[:key]])
+      end
+      indexes
+    end
+  end
+end

data/lib/mysql_warmup/version.rb

@@ -0,0 +1,3 @@
+module MysqlWarmup
+  VERSION = '0.0.1'.freeze
+end

data/lib/mysql_warmup/warmer.rb

@@ -0,0 +1,95 @@
+require 'mysql'
+module MysqlWarmup
+  class Warmer
+    ALL_DATABASE      = 'all'.freeze
+    EXCLUDE_DATABASES = %w(information_schema mysql performance_schema sys).freeze
+    PREVENT_VARIABLES = [:@host, :@username, :@password, :@database, :@port].freeze
+
+    def initialize(host, username, password, port = 3306, database = 'all')
+      @host     = host
+      @username = username
+      @password = password
+      @database = database
+      @port     = port
+
+      @connector = if warmup_all?
+                     Mysql.new(@host, @username, @password, '', @port)
+                   else
+                     Mysql.new(@host, @username, @password, @database, @port)
+                   end
+    end
+
+    def warmup
+      warmup_all? ? warmup_all_dbs : warmup_only
+    end
+
+    # Prevent inspection object
+    def inspect
+      "#<MysqlWarmup::Warmer:#{object_id}>"
+    end
+
+    def instance_variable_get(*several_variants)
+      raise 'Not allow to view this variable' if PREVENT_VARIABLES.include?(several_variants[0])
+      super
+    end
+
+    private
+
+    def warmup_all_dbs
+      @connector.list_dbs.each do |db|
+        next if EXCLUDE_DATABASES.include?(db)
+        MysqlWarmup::Warmer.new(@host, @username, @password, @port, db).warmup
+      end
+    end
+
+    def warmup_only
+      write_log(">>>>>>> START WARMUP FOR DB: #{@database} <<<<<<")
+      tables = @connector.query('show tables')
+      table  = tables.fetch_row
+      while table
+        # Fetch fields infos
+        fields_infos = fetch_fields_infos(table[0])
+
+        table_instance = MysqlWarmup::Table.new(table[0], fields_infos)
+        write_log("START WARMUP FOR TABLE:   `#{@database}`.`#{table[0]}`")
+        table_instance.indexes.each do |i|
+          touch(i.query_string)
+        end
+        write_log("SUCCESS WARMUP FOR TABLE: `#{@database}`.`#{table[0]}`\n\n")
+
+        # Continue fetching table
+        table = tables.fetch_row
+      end
+      write_log("+++++++ SUCCESS WARMUP FOR DB: #{@database} +++++++\n\n")
+    rescue Mysql::Error => e
+      write_log("ERROR: ----------- #{e.message}")
+      write_log("BACKTRACE: ------- #{e.backtrace[0, 5]}")
+    ensure
+      @connector.close if @connector
+    end
+
+    def fetch_fields_infos(table_name)
+      fields_infos = []
+      fields       = @connector.query("describe `#{table_name}`")
+      field        = fields.fetch_row
+      while field
+        fields_infos << field
+        field = fields.fetch_row
+      end
+      fields_infos
+    end
+
+    def touch(query_string)
+      # write_log(query_string)
+      @connector.query(query_string)
+    end
+
+    def write_log(log_msg)
+      MysqlWarmup::Logger.write(log_msg)
+    end
+
+    def warmup_all?
+      @database.downcase == ALL_DATABASE
+    end
+  end
+end

data/lib/mysql_warmup.rb

@@ -0,0 +1,7 @@
+module MysqlWarmup; end
+
+require 'mysql_warmup/logger'
+require 'mysql_warmup/index'
+require 'mysql_warmup/table'
+require 'mysql_warmup/warmer'
+require 'mysql_warmup/version'

data/mysql-warmup.gemspec

@@ -0,0 +1,35 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mysql_warmup/version'
+
+Gem::Specification.new do |s|
+  s.name     = 'mysql-warmup'
+  s.version  = MysqlWarmup::VERSION
+  s.author   = 'Manh Dao Van'
+  s.email    = 'manhdaovan@gmail.com'
+  s.homepage = 'https://github.com/manhdaovan/mysql_warmup'
+  s.license  = 'MIT'
+
+  s.summary     = 'A command line tool to warm up mysql instance after reboot or startup'
+  s.description = <<-eos
+    When you've just created new slave instance, first requests to DB (with InnoDB storage engine)
+    will be hit on disk instead of buffer poll. So, the requests will be slow down.
+    You can use this tool for warming up buffer poll before the first requests come.
+    Please see document for other cases.
+  eos
+
+  s.platform              = Gem::Platform::RUBY
+  s.required_ruby_version = '>= 2.0.0'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test}/*`.split("\n")
+  s.require_paths = ['lib']
+
+  s.executables        = s.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+  s.default_executable = 'mysql-warmup'
+  s.bindir             = 'bin'
+
+  s.add_dependency('mysql')
+  s.add_development_dependency('rake')
+end

data/test/helper.rb

@@ -0,0 +1,7 @@
+def query_template
+  'select %s from %s where %s'.freeze
+end
+
+def build_query(*params)
+  format(query_template, *params)
+end

data/test/test_index.rb

@@ -0,0 +1,20 @@
+require 'test/unit'
+require 'mysql_warmup'
+require 'helper'
+
+class TestIndex < Test::Unit::TestCase
+  def test_query_string
+    types = %w(VARCHAR TEXT BLOB INT DATE OTHERS)
+    # Index for primary
+    index = MysqlWarmup::Index.new('table_name', 'col_name', 'INT', 'PRI')
+    assert_equal(build_query('sum(`table_name`.`col_name`)', '`table_name`', '1'),
+                 index.query_string)
+
+    # Index for other fields
+    types.each do |type|
+      index = MysqlWarmup::Index.new('table_name', 'col_name', type, '')
+      assert_equal(build_query('count(*)', '`table_name`', "`table_name`.`col_name` LIKE '%0%'"),
+                   index.query_string)
+    end
+  end
+end

data/test/test_logger.rb

@@ -0,0 +1,13 @@
+require 'test/unit'
+require 'mysql_warmup'
+
+class TestLogger < Test::Unit::TestCase
+  def test_logger_output
+    orig_stdout = $stdout
+    $stdout     = StringIO.new
+    MysqlWarmup::Logger.write('log output')
+    assert_equal(true, !($stdout.string =~ /log output/).nil?)
+  ensure
+    $stdout = orig_stdout
+  end
+end

data/test/test_mysql_warmup.rb

@@ -0,0 +1,5 @@
+require 'test/unit'
+require 'mysql_warmup'
+
+class TestMysqlWarmup < Test::Unit::TestCase
+end

data/test/test_table.rb

@@ -0,0 +1,14 @@
+require 'test/unit'
+require 'mysql_warmup'
+
+class TestTable < Test::Unit::TestCase
+  def test_indexes_of_table
+    table_name  = 'table_name'
+    fields_info = [%w(field type null key default extra)] * 5
+    table       = MysqlWarmup::Table.new(table_name, fields_info)
+    assert_equal(5, table.indexes.size)
+    table.indexes.each do |table_index|
+      assert_equal(true, table_index.is_a?(MysqlWarmup::Index))
+    end
+  end
+end

data/test/test_warmer.rb

@@ -0,0 +1,21 @@
+require 'test/unit'
+require 'mysql_warmup'
+
+class TestWarmer < Test::Unit::TestCase
+  def test_init_and_prevent_variable
+    Mysql.class_eval do
+      def self.new(*_arg)
+        'Mysql Instance'
+      end
+    end
+    warmer = MysqlWarmup::Warmer.new('localhost', 'db_user', 'db_password')
+    MysqlWarmup::Warmer::PREVENT_VARIABLES.each do |p_v|
+      begin
+        warmer.instance_variable_get(p_v)
+      rescue => e
+        assert_equal('Not allow to view this variable', e.message)
+      end
+    end
+    assert_equal('Mysql Instance', warmer.instance_variable_get(:@connector))
+  end
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: mysql-warmup
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Manh Dao Van
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-01-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mysql
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |2
+      When you've just created new slave instance, first requests to DB (with InnoDB storage engine)
+      will be hit on disk instead of buffer poll. So, the requests will be slow down.
+      You can use this tool for warming up buffer poll before the first requests come.
+      Please see document for other cases.
+email: manhdaovan@gmail.com
+executables:
+- console
+- mysql-warmup
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- bin/console
+- bin/mysql-warmup
+- lib/mysql_warmup.rb
+- lib/mysql_warmup/index.rb
+- lib/mysql_warmup/logger.rb
+- lib/mysql_warmup/table.rb
+- lib/mysql_warmup/version.rb
+- lib/mysql_warmup/warmer.rb
+- mysql-warmup.gemspec
+- test/helper.rb
+- test/test_index.rb
+- test/test_logger.rb
+- test/test_mysql_warmup.rb
+- test/test_table.rb
+- test/test_warmer.rb
+homepage: https://github.com/manhdaovan/mysql_warmup
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: A command line tool to warm up mysql instance after reboot or startup
+test_files: []