mysql-warmup 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 528bec2ec40aebd23618e7b565ad1ebc56fecc2e
-  data.tar.gz: 826ada4963a4bbaeb26915ffe68ece199f994650
+  metadata.gz: 260186eb7857d1532a09cb2deb0cadb6affe54f2
+  data.tar.gz: 078a46dd9fab6acfa5b05caf7f315653bf3b6163
 SHA512:
-  metadata.gz: b372e3fe62daddca167677763348853b0f7067188056fa076e16b3db0caa6e7688cb8dd1552521658fb41d8e965d877005d6466c365a9f83bac8840e97a6587b
-  data.tar.gz: c5ef0652894e93a7542f0fd349b81aae5f6511ae1a53686a0ea35265f0c08e7e36cdd5d7b6c30ffc4ba6d974e0f7e70a34b89c0f9cffb04fe65f720fb0a99c7a
+  metadata.gz: 71e52cf2de14ee80d2479cd1fda6f61a346ae7dfd296d5c3bfdf82c7f5f98681aa7981f7b42c8b2a8ea9cd78fa90680ef4b79cdad3025f8740dbc9e5541314a4
+  data.tar.gz: f9cd930ebaa0c9a3dee6d4a96c8bd69687769a80fd933781c4c1e4df4555e222ad64c6f3d66f0d241351163d865dd0086e1c9d04921abcbe1c7257bf2fbc2181

data/CHANGELOG.md

@@ -1,4 +1,9 @@
 # Changelog
+### v.0.0.3 (2017/01/15)
+* Add default value 'localhost' to -h param
+* Improve touching primary by `count(*) where non_index_file = 0` instead of `sum(id)` <br/>
+ [Reason](./CHANGE_SUM_TO_COUNT.md)
+
 
 ### v.0.0.2 (2017/01/14)
 * Change check type_primary? condition to regardless to field type

data/CHANGE_SUM_TO_COUNT.md

@@ -0,0 +1,262 @@
+# Background
+When execute command to load primary key to buffer pool, what command should we do?
+`select count(*) from table_name` or `select count(*) from table_name where non_index_column = 0` or `select sum(primary_key) from table_name` ?
+
+# Testing
+### Testing environment
+* Mysql version `5.7.17`
+* Buffer pool configuration:
+    ```
+    mysql> show variables like '%buffer_pool%';
+    +-------------------------------------+----------------+
+    | Variable_name                       | Value          |
+    +-------------------------------------+----------------+
+    | innodb_buffer_pool_chunk_size       | 134217728      |
+    | innodb_buffer_pool_dump_at_shutdown | OFF            |
+    | innodb_buffer_pool_dump_now         | OFF            |
+    | innodb_buffer_pool_dump_pct         | 25             |
+    | innodb_buffer_pool_filename         | ib_buffer_pool |
+    | innodb_buffer_pool_instances        | 1              |
+    | innodb_buffer_pool_load_abort       | OFF            |
+    | innodb_buffer_pool_load_at_startup  | OFF            |
+    | innodb_buffer_pool_load_now         | OFF            |
+    | innodb_buffer_pool_size             | 134217728      |
+    +-------------------------------------+----------------+
+    10 rows in set (0.01 sec)
+    ```
+
+* Database `test_mysql_dump` has only table `test_data` with 10M rows, configurations as below:
+
+    ```
+    mysql> select table_name, engine from information_schema.tables where table_schema = 'test_mysql_warmup';
+    +------------+--------+
+    | table_name | engine |
+    +------------+--------+
+    | test_data  | InnoDB |
+    +------------+--------+
+    1 row in set (0.00 sec)
+    ```
+
+    and
+    ```
+    mysql> desc test_data;
+    +-------------+--------------+------+-----+---------+-------+
+    | Field       | Type         | Null | Key | Default | Extra |
+    +-------------+--------------+------+-----+---------+-------+
+    | id          | int(11)      | NO   | PRI | NULL    |       |
+    | random_str  | varchar(255) | YES  |     | NULL    |       |
+    | created_at  | varchar(255) | YES  |     | NULL    |       |
+    | random_str1 | varchar(255) | YES  |     | NULL    |       |
+    | random_str2 | varchar(255) | YES  |     | NULL    |       |
+    | random_str3 | varchar(255) | YES  |     | NULL    |       |
+    +-------------+--------------+------+-----+---------+-------+
+    6 rows in set (0.00 sec)
+    ```
+
+### Test scenario (Restart mysql before each testing with each command)
+1. **Compare above 3 command when table `test_data` has only primary key as index**:
+    * Testing with command: `select sum(id) from test_data;`:
+
+        ```
+        mysql> select sum(id) from test_data;
+        +----------------+
+        | sum(id)        |
+        +----------------+
+        | 50000005000000 |
+        +----------------+
+        1 row in set (12.46 sec)
+        ```
+        **Buffer poll infos**
+        ```
+        mysql> select table_name as Table_Name, index_name as Index_Name, count(*) as Page_Count, sum(data_size)/1024/1024 as Size_in_MB from information_schema.innodb_buffer_page group by table_name, index_name order by Size_in_MB desc limit 5;
+        +---------------------------------+------------+------------+--------------+
+        | Table_Name                      | Index_Name | Page_Count | Size_in_MB   |
+        +---------------------------------+------------+------------+--------------+
+        | `test_mysql_warmup`.`test_data` | PRIMARY    |       7940 | 106.90351486 |
+        | `SYS_TABLES`                    | CLUST_IND  |         86 |   1.20526028 |
+        | `SYS_COLUMNS`                   | CLUST_IND  |          3 |   0.02067661 |
+        | `mysql`.`innodb_index_stats`    | PRIMARY    |          2 |   0.01506424 |
+        | `SYS_FIELDS`                    | CLUST_IND  |          1 |   0.00902271 |
+        +---------------------------------+------------+------------+--------------+
+        5 rows in set (0.07 sec)
+        ```
+
+        => **Primary key** is loaded with **7940 pages**
+
+    * Testing with command: `select count(*) from test_data;`:
+
+        ```
+        mysql> select count(*) from test_data;
+        | count(*) |
+        +----------+
+        | 10000000 |
+        +----------+
+        1 row in set (11.68 sec)
+        ```
+
+        **Buffer poll infos**
+        ```
+        mysql> select table_name as Table_Name, index_name as Index_Name, count(*) as Page_Count, sum(data_size)/1024/1024 as Size_in_MB from information_schema.innodb_buffer_page group by table_name, index_name order by Size_in_MB desc limit 5;
+        +---------------------------------+------------+------------+--------------+
+        | Table_Name                      | Index_Name | Page_Count | Size_in_MB   |
+        +---------------------------------+------------+------------+--------------+
+        | `test_mysql_warmup`.`test_data` | PRIMARY    |       7940 | 106.90351486 |
+        | `SYS_TABLES`                    | CLUST_IND  |         86 |   1.20526028 |
+        | `SYS_COLUMNS`                   | CLUST_IND  |          3 |   0.02067661 |
+        | `mysql`.`innodb_index_stats`    | PRIMARY    |          2 |   0.01506424 |
+        | `SYS_FIELDS`                    | CLUST_IND  |          1 |   0.00902271 |
+        +---------------------------------+------------+------------+--------------+
+        5 rows in set (0.08 sec)
+        ```
+        => **Primary key** is loaded with **7940 pages**
+
+    * Testing with command: `select count(*) from test_data where random_str = 0 or random_str = '0';`:
+
+        ```
+        mysql> select count(*) from test_data where random_str = 0 or random_str = '0';
+        +----------+
+        | count(*) |
+        +----------+
+        | 10000000 |
+        +----------+
+        1 row in set, 65535 warnings (19.08 sec)
+        ```
+
+        **Buffer pool infos**
+        ```
+        mysql> select table_name as Table_Name, index_name as Index_Name, count(*) as Page_Count, sum(data_size)/1024/1024 as Size_in_MB from information_schema.innodb_buffer_page group by table_name, index_name order by Size_in_MB desc limit 5;
+        +---------------------------------+------------+------------+--------------+
+        | Table_Name                      | Index_Name | Page_Count | Size_in_MB   |
+        +---------------------------------+------------+------------+--------------+
+        | `test_mysql_warmup`.`test_data` | PRIMARY    |       8025 | 108.04996300 |
+        | `SYS_COLUMNS`                   | CLUST_IND  |          3 |   0.02067661 |
+        | `SYS_FIELDS`                    | CLUST_IND  |          1 |   0.00906658 |
+        | `SYS_TABLES`                    | CLUST_IND  |          2 |   0.00766659 |
+        | `SYS_INDEXES`                   | CLUST_IND  |          1 |   0.00720882 |
+        +---------------------------------+------------+------------+--------------+
+        ```
+        => **Primary key** is loaded with **8025 pages**
+
+2. **Compare above 3 command when table `test_data` has primary key `id` and field `random_str` as index**
+    * First, add index to field `random_str`:
+
+        ```
+        mysql> create index test_data_random_str_index on test_data(random_str);
+        Query OK, 0 rows affected (52.24 sec)
+        Records: 0  Duplicates: 0  Warnings: 0
+
+        mysql> desc test_data;
+        +-------------+--------------+------+-----+---------+-------+
+        | Field       | Type         | Null | Key | Default | Extra |
+        +-------------+--------------+------+-----+---------+-------+
+        | id          | int(11)      | NO   | PRI | NULL    |       |
+        | random_str  | varchar(255) | YES  | MUL | NULL    |       |
+        | created_at  | varchar(255) | YES  |     | NULL    |       |
+        | random_str1 | varchar(255) | YES  |     | NULL    |       |
+        | random_str2 | varchar(255) | YES  |     | NULL    |       |
+        | random_str3 | varchar(255) | YES  |     | NULL    |       |
+        +-------------+--------------+------+-----+---------+-------+
+        6 rows in set (0.00 sec)
+
+        mysql> show index from test_data;
+        +-----------+------------+----------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+
+        | Table     | Non_unique | Key_name                   | Seq_in_index | Column_name | Collation | Cardinality | Sub_part | Packed | Null | Index_type | Comment | Index_comment |
+        +-----------+------------+----------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+
+        | test_data |          0 | PRIMARY                    |            1 | id          | A         |     9943860 |     NULL | NULL   |      | BTREE      |         |               |
+        | test_data |          1 | test_data_random_str_index |            1 | random_str  | A         |     9943860 |     NULL | NULL   | YES  | BTREE      |         |               |
+        +-----------+------------+----------------------------+--------------+-------------+-----------+-------------+----------+--------+------+------------+---------+---------------+
+        2 rows in set (0.00 sec)
+        ```
+
+    * Testing with command: `select sum(id) from test_data;`:
+
+        ```
+        mysql> select sum(id) from test_data;
+        +----------------+
+        | sum(id)        |
+        +----------------+
+        | 50000005000000 |
+        +----------------+
+        1 row in set (3.28 sec)
+        ```
+
+        **Buffer pool infos**
+        ```
+        mysql> select table_name as Table_Name, index_name as Index_Name, count(*) as Page_Count, sum(data_size)/1024/1024 as Size_in_MB from information_schema.innodb_buffer_page group by table_name, index_name order by Size_in_MB desc limit 5;
+        +---------------------------------+----------------------------+------------+--------------+
+        | Table_Name                      | Index_Name                 | Page_Count | Size_in_MB   |
+        +---------------------------------+----------------------------+------------+--------------+
+        | `test_mysql_warmup`.`test_data` | test_data_random_str_index |       7877 | 120.96562290 |
+        | `SYS_TABLES`                    | CLUST_IND                  |         86 |   1.20540619 |
+        | `SYS_COLUMNS`                   | CLUST_IND                  |          3 |   0.02067661 |
+        | `SYS_FIELDS`                    | CLUST_IND                  |          1 |   0.00906658 |
+        | `SYS_INDEXES`                   | CLUST_IND                  |          1 |   0.00720882 |
+        +---------------------------------+----------------------------+------------+--------------+
+        5 rows in set (0.07 sec)
+        ```
+
+        => **test_data_random_str_index** is loaded to buffer pool instead of `primary key`, with **7877 pages**
+
+    * Testing with command: `select count(*) from test_data;`:
+
+        ```
+        mysql> select count(*) from test_data;
+        +----------+
+        | count(*) |
+        +----------+
+        | 10000000 |
+        +----------+
+        1 row in set (15.43 sec)
+        ```
+        **Buffer pool info**
+        ```
+        mysql> select table_name as Table_Name, index_name as Index_Name, count(*) as Page_Count, sum(data_size)/1024/1024 as Size_in_MB from information_schema.innodb_buffer_page group by table_name, index_name order by Size_in_MB desc limit 5;
+        +---------------------------------+------------+------------+--------------+
+        | Table_Name                      | Index_Name | Page_Count | Size_in_MB   |
+        +---------------------------------+------------+------------+--------------+
+        | `test_mysql_warmup`.`test_data` | PRIMARY    |       7940 | 106.90351486 |
+        | `SYS_TABLES`                    | CLUST_IND  |         86 |   1.20526028 |
+        | `SYS_COLUMNS`                   | CLUST_IND  |          3 |   0.02067661 |
+        | `SYS_FIELDS`                    | CLUST_IND  |          1 |   0.00906658 |
+        | `SYS_INDEXES`                   | CLUST_IND  |          1 |   0.00720882 |
+        +---------------------------------+------------+------------+--------------+
+        5 rows in set (0.06 sec)
+        ```
+        => **Primary key** is loaded into buffer pool with **7940 pages**.
+
+    * Testing with command: `select count(*) from test_data where random_str2 = 0 or random_str2 = '0'` (with `random_str2` is `non-index column`):
+
+        ```
+        mysql> select count(*) from test_data where random_str2 = '0' or random_str2 = 0;
+        | count(*) |
+        +----------+
+        |        0 |
+        +----------+
+        1 row in set (14.42 sec)
+        ```
+        **Buffer pool infos**
+        ```
+        mysql> select table_name as Table_Name, index_name as Index_Name, count(*) as Page_Count, sum(data_size)/1024/1024 as Size_in_MB from information_schema.innodb_buffer_page group by table_name, index_name order by Size_in_MB desc limit 5;
+        +---------------------------------+------------+------------+--------------+
+        | Table_Name                      | Index_Name | Page_Count | Size_in_MB   |
+        +---------------------------------+------------+------------+--------------+
+        | `test_mysql_warmup`.`test_data` | PRIMARY    |       8039 | 108.24720383 |
+        | `SYS_TABLES`                    | CLUST_IND  |        147 |   2.08744812 |
+        | NULL                            | NULL       |          5 |   0.00000000 |
+        +---------------------------------+------------+------------+--------------+
+        3 rows in set (0.06 sec)
+        ```
+        => **Primary key** is loaded with **8039 pages** (greater than **7940 pages**)
+
+# Conclusion:
+1. Table with only primary key:
+    * `select count(*) from table_name where non_index_column = 0 or non_index_column = '0'` has the greatest pages loaded.
+    * `select count(*) from table_name` and `select sum(primary_key) from table_name` has the same result, and the loaded pages is less than query has `where non_index_column = 0 or non_index_column = '0'` condition.
+
+2. Table with primary and other index(es):
+    * `select count(*) from table_name`: **primary_key** will be loaded.
+    * `select sum(primary_key) from table_name`: **other_index** will be loaded.
+    * `select count(*) from table_name where non_index_column = 0 or non_index_column = '0'`: **primary_key** will be loaded with the greatest.
+
+
+

data/bin/mysql-warmup

@@ -9,10 +9,10 @@ def print_help
   puts "mysql-warmup, by ManhDV - version #{MysqlWarmup::VERSION}"
   puts 'Website: https://github.com/manhdaovan/mysql_warmup'
   puts
-  puts 'Usage: mysql-warmup -h host-or-ip -u username <options>'
+  puts 'Usage: mysql-warmup -u username <options>'
   puts
   puts 'Input options:'
-  puts '-h host      : Host or ip of mysql instance'
+  puts '-h host      : Host or ip of mysql instance. Default is localhost'
   puts '-u username  : Username to access mysql instance'
   puts '-d database  : Database to warmup.'
   puts '               Default to all databases exclude information_schema, mysql, performance_schema, sys'
@@ -37,7 +37,7 @@ def params_valid?(options)
   valid
 end
 
-options = { show_version: false, help: false, port: 3306, database: 'all' }
+options = { show_version: false, help: false, port: 3306, database: 'all', host: 'localhost' }
 
 begin
   OptionParser.new do |opts|

data/lib/mysql_warmup/index.rb

@@ -3,20 +3,25 @@ module MysqlWarmup
     QUERY_TEMPLATE = 'select %s from %s where %s'.freeze
     attr_reader :query_string
 
-    def initialize(table_name, column_name, column_type, column_key)
+    def initialize(table_name, column_name, column_type, column_key, non_index_field = nil)
       @table_name   = table_name
       @column_name  = column_name
       @column_type  = column_type
       @column_key   = column_key
       @query_string = build_query_string(@table_name, @column_name,
-                                         @column_type, @column_key)
+                                         @column_type, @column_key, non_index_field)
     end
 
     private
 
-    def build_query_string(table_name, column_name, column_type, column_key)
+    def build_query_string(table_name, column_name, column_type, column_key, non_index_field = nil)
       if type_primary?(column_key)
-        format_query("sum(`#{table_name}`.`#{column_name}`)", "`#{table_name}`", '1')
+        if non_index_field.nil?
+          format_query('count(*)', "`#{table_name}`", '1')
+        else
+          field_name = "`#{table_name}`.`#{non_index_field}`"
+          format_query('count(*)', "`#{table_name}`", "#{field_name} = 0 or #{field_name} = '0'")
+        end
       elsif type_integer?(column_type)
         format_query('count(*)', "`#{table_name}`", "`#{table_name}`.`#{column_name}` LIKE '%0%'")
       elsif type_var_char?(column_type)

data/lib/mysql_warmup/table.rb

@@ -20,13 +20,17 @@ module MysqlWarmup
     private
 
     def build_index
-      indexes_infos = @field_infos.select { |v| !v[DESC_TABLE_STRUCTURE[:key]].empty? }
-      indexes       = []
+      indexes_infos    = []
+      non_indexes_info = []
+      @field_infos.each { |v| v[DESC_TABLE_STRUCTURE[:key]].empty? ? non_indexes_info << v : indexes_infos << v }
+      non_index_field = non_indexes_info.empty? ? nil : non_indexes_info[0][DESC_TABLE_STRUCTURE[:field]]
+      indexes         = []
       indexes_infos.each do |index_info|
         indexes << MysqlWarmup::Index.new(@table_name,
                                           index_info[DESC_TABLE_STRUCTURE[:field]],
                                           index_info[DESC_TABLE_STRUCTURE[:type]],
-                                          index_info[DESC_TABLE_STRUCTURE[:key]])
+                                          index_info[DESC_TABLE_STRUCTURE[:key]],
+                                          non_index_field)
       end
       indexes
     end

data/lib/mysql_warmup/version.rb

@@ -1,3 +1,3 @@
 module MysqlWarmup
-  VERSION = '0.0.2'.freeze
+  VERSION = '0.0.3'.freeze
 end

data/test/test_index.rb

@@ -6,9 +6,15 @@ class TestIndex < Test::Unit::TestCase
   def test_query_string
     types = %w(VARCHAR TEXT BLOB INT DATE OTHERS)
     # Index for primary
+    # 1. without any non-index field
     index = MysqlWarmup::Index.new('table_name', 'col_name', 'INT', 'PRI')
-    assert_equal(build_query('sum(`table_name`.`col_name`)', '`table_name`', '1'),
+    assert_equal(build_query('count(*)', '`table_name`', '1'),
                  index.query_string)
+    # 2. with non-index field
+    index2 = MysqlWarmup::Index.new('table_name', 'col_name', 'INT', 'PRI', 'non_index')
+    field_name = '`table_name`.`non_index`'
+    assert_equal(build_query('count(*)', '`table_name`', "#{field_name} = 0 or #{field_name} = '0'"),
+                 index2.query_string)
 
     # Index for other fields
     types.each do |type|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mysql-warmup
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Manh Dao Van
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-14 00:00:00.000000000 Z
+date: 2017-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql
@@ -51,6 +51,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - CHANGELOG.md
+- CHANGE_SUM_TO_COUNT.md
 - README.md
 - Rakefile
 - bin/console
@@ -62,6 +63,7 @@ files:
 - lib/mysql_warmup/version.rb
 - lib/mysql_warmup/warmer.rb
 - mysql-warmup-0.0.1.gem
+- mysql-warmup-0.0.2.gem
 - mysql-warmup.gemspec
 - test/helper.rb
 - test/test_index.rb