zdm 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c8db3104b8d62f7bed12337cff889620888e79f
-  data.tar.gz: d517e08f0e74b0689aae83b199c6ec41974d7dd0
+  metadata.gz: 5e342d256ff439757a08cda9ad15897c90278527
+  data.tar.gz: 72894ff47ca9aaca42c6cd0a3b07bc3d0f492490
 SHA512:
-  metadata.gz: 8f93e0daed451fcac1f51f0c128576967f3cfe397e5c4ed75a7ea82691c86550f3b277a82506712023fc04d49167233092771e92f802c6cf245f3bfec59e023d
-  data.tar.gz: bc9777cbb66ae16c10feaeb80263e6e43df1b0c43234de7f3d93754f3e9e1f3271b21911d1f7a09317278b068c69d79e996bcc63546033c817f0b15996e2603a
+  metadata.gz: a57d77a2afd53a03508aac7a253d7521ced90318e52ce0c8583e8a2cc5960804b4631f4e94fb6e6424599bc0b7b2c8c578beea3c2ec6124203f1b0232ae11514
+  data.tar.gz: ff0c63a2eab00afa7fc4ec348fd1088348cdec28ecd23214b42818d2c3c6e9da01b96c452257292248234f30a983ba60e6b8e1cf031a3e778d132ebda75103fc

data/README.md

@@ -8,6 +8,8 @@ Instead of using outfiles we follow [lhm](https://github.com/soundcloud/lhm)'s a
 
 The code is the readme. If you donot grok the code then you really should not use this.
 
+See also documentation from mysql on [Online Status for DDL Operations](https://dev.mysql.com/doc/refman/5.7/en/innodb-create-index-overview.html#innodb-online-ddl-summary-grid).
+
 Install
 =======
 

data/gemfiles/4.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.2)
+    zdm (1.0.3)
       activerecord (>= 4.0)
 
 GEM

data/gemfiles/4.2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.2)
+    zdm (1.0.3)
       activerecord (>= 4.0)
 
 GEM

data/gemfiles/5.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    zdm (1.0.2)
+    zdm (1.0.3)
       activerecord (>= 4.0)
 
 GEM

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Zdm
-  VERSION = '1.0.2'
+  VERSION = '1.0.3'
 end

data/lib/zdm.rb

@@ -76,12 +76,12 @@ module Zdm
       validate
       set_session_lock_wait_timeouts
       cleanup
-      create_destination_table
-      drop_destination_indexes
+      create_copy_table
+      drop_copy_indexes
       apply_ddl_statements
       create_triggers
       batched_copy
-      create_destination_indexes
+      create_copy_indexes
       atomic_switcharoo!
     ensure
       cleanup
@@ -124,7 +124,7 @@ module Zdm
         end
       end
 
-      def create_destination_table
+      def create_copy_table
         execute('CREATE TABLE `%s` LIKE `%s`' % [table.copy, table.origin])
       end
 
@@ -180,26 +180,39 @@ module Zdm
         "zdmt_#{trigger_type}_#{table.origin}"[0...64]
       end
 
-      # Drop indexes to speed up batched_copy
-      def drop_destination_indexes
+      # Drop indexes to speed up batched_copy.
+      #
+      # "Online DDL support for adding secondary indexes means that you can
+      # generally speed the overall process of creating and loading a table
+      # and associated indexes by creating the table without any secondary
+      # indexes, then adding the secondary indexes after the data is loaded."
+      # https://dev.mysql.com/doc/refman/5.7/en/innodb-create-index-overview.html#idm140602200949744
+      def drop_copy_indexes
         @indexes = connection.indexes(table.copy).reject(&:unique)
         @indexes.each do |index_def|
           execute('ALTER TABLE `%s` DROP INDEX `%s`' % [table.copy, index_def.name])
         end
       end
 
-      # Recreate the indexes previously dropped
-      def create_destination_indexes
-        @indexes.each do |index_def|
-          opts = { name: index_def.name, using: index_def.using }
-
-          opts[:length] = if index_def.lengths.is_a?(Hash)
+      # Recreate the indexes previously dropped, using 1 statement so the table
+      # is read through once.
+      def create_copy_indexes
+        return if @indexes.empty?
+        indexes = @indexes.map do |index_def|
+          lengths = if index_def.lengths.is_a?(Hash)
             index_def.lengths
           elsif index_def.lengths.compact.any?
             Hash[index_def.columns.map.with_index { |col, idx| [col, index_def.lengths[idx]] }]
           end
-          connection.add_index(table.copy, index_def.columns, opts)
+          index_columns = index_def.columns.map.with_index do |col, idx|
+            index_col_name = '`%s`' % col
+            column_length = lengths && lengths[col]
+            index_col_name << '(%s)' % column_length if column_length
+            index_col_name
+          end
+          "ADD INDEX `#{index_def.name}` (#{index_columns.join(',')}) USING #{index_def.using}"
         end
+        execute('ALTER TABLE `%s` %s' % [table.copy, indexes.join(', ')])
       end
 
       BATCH_SIZE = 40_000
@@ -216,6 +229,12 @@ module Zdm
 
         insert_columns = common_columns.map {|c| "`#{c}`"}.join(', ')
         select_columns = common_columns.map {|c| "`#{table.origin}`.`#{c}`"}.join(', ')
+        sql = <<-SQL.squish
+          INSERT IGNORE INTO `#{table.copy}` (#{insert_columns})
+          SELECT #{select_columns}
+          FROM `#{table.origin}`
+          WHERE `#{table.origin}`.`id` BETWEEN %s AND %s
+        SQL
 
         batch_size = BATCH_SIZE
         batch_end = min - 1
@@ -225,12 +244,7 @@ module Zdm
           batch_end = [batch_start + batch_size - 1, max].min
           start_batch_time = Time.now
 
-          execute(<<-SQL.squish)
-            INSERT IGNORE INTO `#{table.copy}` (#{insert_columns})
-            SELECT #{select_columns}
-            FROM `#{table.origin}`
-            WHERE `#{table.origin}`.`id` BETWEEN #{batch_start} AND #{batch_end}
-          SQL
+          execute(sql % [batch_start, batch_end])
 
           if $exit
             write('Received SIGTERM, exiting...')

data/spec/spec_helper.rb

@@ -26,6 +26,7 @@ ActiveRecord::Schema.define version: 0 do
   end
   add_index(:people, :name, unique: true)
   add_index(:people, [:account_id, :code], length: {account_id: nil, code: 191})
+  add_index(:people, :created_at)
 
   create_table :people_teams, id: false, force: true do |t|
     t.integer :team_id, null: false
@@ -36,3 +37,5 @@ end
 ActiveRecord::Base.connection.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (10,'foo','bar','2017-03-01 23:59:59')])
 ActiveRecord::Base.connection.execute(%[INSERT INTO people(account_id, name, code, created_at) VALUES (20,'foo2','bar2','2017-03-02 23:59:59')])
 
+# ActiveRecord::Base.logger = Logger.new($stdout)
+

data/spec/zdm_spec.rb

@@ -44,7 +44,8 @@ describe Zdm do
         `created_at` datetime DEFAULT NULL,
         `test` varchar(32) COLLATE utf8_unicode_ci DEFAULT 'foo',
         PRIMARY KEY (`id`), UNIQUE KEY `index_people_on_name` (`name`),
-        KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE
+        KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE,
+        KEY `index_people_on_created_at` (`created_at`) USING BTREE
       ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
     EOS
 
@@ -77,7 +78,8 @@ describe Zdm do
           `code` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
           `created_at` datetime DEFAULT NULL, PRIMARY KEY (`id`),
           UNIQUE KEY `index_people_on_name` (`name`),
-          KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE
+          KEY `index_people_on_account_id_and_code` (`account_id`,`code`(191)) USING BTREE,
+          KEY `index_people_on_created_at` (`created_at`) USING BTREE
         ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
       EOS
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zdm
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - ITRP Institute, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-03 00:00:00.000000000 Z
+date: 2017-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord