lhm-shopify 3.3.5 → 3.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c187c9f0e9bd183d5eb227f83ba9cce6a57ee5b03e4732312470a737f2d8a13
-  data.tar.gz: 54797e75f27f2f174ffd0eb07cd6d1fad57a9d16542e5baa0a56cb1130fcd88a
+  metadata.gz: cfb1802db81b2673ea5e52acec84b4e457a8d5debb74f5391fcb282bf1fc8484
+  data.tar.gz: 88abd0ae0484efa69777b0284c253788996b2148c0cb61fc035c0ccd7d49627c
 SHA512:
-  metadata.gz: 9f11850e292435adb700695686887945d729ec8ec3b2d4b658fd1d6dd297f198e17711b26f2f1533cca0a58cdcb5f45a87bdbb1741f0cd988b57dfb47bd90d5b
-  data.tar.gz: ba2e873e51070ac3658644f90c7f54a1680707e8aa48db2d844fdcbfd2de83a5a96abbc0521eae9e55894d5e93ee399f9327df4c56f6b95945e381894e93ff6a
+  metadata.gz: 4b98a803678e1ab543997d4e6f9031b02511324293c2d97b3f94938ae6b701e55c122633ab6044920363c56d9746e35f9b131433eb65108437c4ffeb8ca2e5ea
+  data.tar.gz: '018c3a179dfdf90f9b5fd4bd3f67636644fc47eda7b0b825aee00330db0f83a59cf6855a5e18105493a89ac0a372ac26fd2c356148282449ca2de84fc56c5179'

data/CHANGELOG.md

@@ -1,4 +1,16 @@
-# 3.3.5 (Jul 5, 2021) 
+# 3.4.1 (Sep 22, 2021)
+
+* Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/108)
+
+# 3.4.0 (Jul 19, 2021)
+
+* Log or raise on unexpected duplicated entry warnings during INSERT IGNORE (https://github.com/Shopify/lhm/pull/100)
+
+# 3.3.6 (Jul 7, 2021)
+
+* Add lhm-shopify.rb to require lhm
+
+# 3.3.5 (Jul 5, 2021)
 
 * Add comment and collate copying to rename_column
 * Publish to rubygems

data/lib/lhm/chunker.rb

@@ -20,6 +20,7 @@ module Lhm
       @connection = connection
       @chunk_finder = ChunkFinder.new(migration, connection, options)
       @options = options
+      @raise_on_warnings = options.fetch(:raise_on_warnings, false)
       @verifier = options[:verifier]
       if @throttler = options[:throttler]
         @throttler.connection = @connection if @throttler.respond_to?(:connection=)
@@ -36,6 +37,8 @@ module Lhm
     end
 
     def execute
+      @start_time = Time.now
+
       return if @chunk_finder.table_empty?
       @next_to_insert = @start
       while @next_to_insert <= @limit || (@start == @limit)
@@ -44,11 +47,26 @@ module Lhm
         verify_can_run
 
         affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @options).insert_and_return_count_of_rows_created
+        expected_rows = top - bottom + 1
+
+        # Only log the chunker progress every 5 minutes instead of every iteration
+        current_time = Time.now
+        if current_time - @start_time > (5 * 60)
+          Lhm.logger.info("Inserted #{affected_rows} rows into the destination table from #{bottom} to #{top}")
+          @start_time = current_time
+        end
+
+        if affected_rows < expected_rows
+          raise_on_non_pk_duplicate_warning
+        end
+
         if @throttler && affected_rows > 0
           @throttler.run
         end
-        @printer.notify(bottom, @limit)
+
         @next_to_insert = top + 1
+        @printer.notify(bottom, @limit)
+
         break if @start == @limit
       end
       @printer.end
@@ -59,6 +77,16 @@ module Lhm
 
     private
 
+    def raise_on_non_pk_duplicate_warning
+      @connection.query("show warnings").each do |level, code, message|
+        unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
+          m = "Unexpected warning found for inserted row: #{message}"
+          Lhm.logger.warn(m)
+          raise Error.new(m) if @raise_on_warnings
+        end
+      end
+    end
+
     def bottom
       @next_to_insert
     end
@@ -83,5 +111,6 @@ module Lhm
       return if @chunk_finder.table_empty?
       @chunk_finder.validate
     end
+
   end
 end

data/lib/lhm/cleanup/current.rb

@@ -63,11 +63,12 @@ module Lhm
             retriable_connection.execute(ddl)
           end
         end
+        Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
+        Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")
       end
 
       def report_ddls
-        puts "The following DDLs would be executed:"
-        ddls.each { |ddl| puts ddl }
+        Lhm.logger.info("The following DDLs would be executed: #{ddls}")
       end
     end
   end

data/lib/lhm/entangler.rb

@@ -94,6 +94,7 @@ module Lhm
           retriable_connection.execute(stmt)
         end
       end
+      Lhm.logger.info("Created triggers on #{@origin.name}")
     end
 
     def after
@@ -102,6 +103,7 @@ module Lhm
           retriable_connection.execute(stmt)
         end
       end
+      Lhm.logger.info("Dropped triggers on #{@origin.name}")
     end
 
     def revert

data/lib/lhm/migrator.rb

@@ -214,6 +214,8 @@ module Lhm
       replacement = %{CREATE TABLE `#{ @origin.destination_name }`}
       stmt = @origin.ddl.gsub(original, replacement)
       @connection.execute(tagged(stmt))
+
+      Lhm.logger.info("Created destination table #{@origin.destination_name}")
     end
 
     def destination_read

data/lib/lhm/printer.rb

@@ -12,26 +12,30 @@ module Lhm
       end
     end
 
-    class Percentage < Base
+    class Percentage
       def initialize
-        super
         @max_length = 0
       end
 
       def notify(lowest, highest)
         return if !highest || highest == 0
+
+        # The argument lowest represents the next_to_insert row id, and highest represents the 
+        # maximum id upto which chunker has to copy the data. 
+        # If all the rows are inserted upto highest, then lowest passed here from chunker was 
+        # highest + 1, which leads to the printer printing the progress > 100%.
+        return if lowest >= highest
+        
         message = "%.2f%% (#{lowest}/#{highest}) complete" % (lowest.to_f / highest * 100.0)
         write(message)
       end
 
       def end
         write('100% complete')
-        @output.write "\n"
       end
 
       def exception(e)
-        write("failed: #{e}")
-        @output.write "\n"
+        Lhm.logger.error("failed: #{e}")
       end
 
       private
@@ -42,7 +46,7 @@ module Lhm
           extra = 0
         end
 
-        @output.write "\r#{message}" + (' ' * extra)
+        Lhm.logger.info(message)
       end
     end
 

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.3.5'
+  VERSION = '3.4.2'
 end

data/lib/lhm-shopify.rb

@@ -0,0 +1 @@
+require "lhm"

data/lib/lhm.rb

@@ -114,17 +114,21 @@ module Lhm
       triggers.each do |trigger|
         connection.execute("drop trigger if exists #{trigger}")
       end
+      logger.info("Dropped triggers #{triggers.join(', ')}")
+
       tables.each do |table|
         connection.execute("drop table if exists #{table}")
       end
+      logger.info("Dropped tables #{tables.join(', ')}")
+
       true
     elsif tables.empty? && triggers.empty?
-      puts 'Everything is clean. Nothing to do.'
+      logger.info('Everything is clean. Nothing to do.')
       true
     else
-      puts "Would drop LHM backup tables: #{tables.join(', ')}."
-      puts "Would drop LHM triggers: #{triggers.join(', ')}."
-      puts 'Run with Lhm.cleanup(true) to drop all LHM triggers and tables, or Lhm.cleanup_current_run(true, table_name) to clean up a specific LHM.'
+      logger.info("Would drop LHM backup tables: #{tables.join(', ')}.")
+      logger.info("Would drop LHM triggers: #{triggers.join(', ')}.")
+      logger.info('Run with Lhm.cleanup(true) to drop all LHM triggers and tables, or Lhm.cleanup_current_run(true, table_name) to clean up a specific LHM.')
       false
     end
   end

data/spec/fixtures/composite_primary_key.ddl

@@ -1,7 +1,6 @@
-
 CREATE TABLE `composite_primary_key` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
   `shop_id` bigint(20) NOT NULL,
-  PRIMARY KEY (`shop_id`,`id`),
+  CONSTRAINT `pk_composite` PRIMARY KEY (`shop_id`,`id`),
   INDEX `index_key_id` (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/composite_primary_key_dest.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `composite_primary_key_dest` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `shop_id` bigint(20) NOT NULL,
+  CONSTRAINT `pk_composite` PRIMARY KEY (`shop_id`,`id`),
+  INDEX `index_key_id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/custom_primary_key_dest.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `custom_primary_key_dest` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `pk` varchar(255),
+  PRIMARY KEY (`pk`),
+  UNIQUE KEY `index_custom_primary_key_on_id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/integration/atomic_switcher_spec.rb

@@ -58,7 +58,7 @@ describe Lhm::AtomicSwitcher do
       switcher.send :define_singleton_method, :atomic_switch do
         'SELECT * FROM nonexistent'
       end
-      -> { switcher.run }.must_raise(ActiveRecord::StatementInvalid)
+      value(-> { switcher.run }).must_raise(ActiveRecord::StatementInvalid)
     end
 
     it "should raise when destination doesn't exist" do
@@ -75,8 +75,8 @@ describe Lhm::AtomicSwitcher do
       switcher.run
 
       slave do
-        data_source_exists?(@origin).must_equal true
-        table_read(@migration.archive_name).columns.keys.must_include 'origin'
+        value(data_source_exists?(@origin)).must_equal true
+        value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
       end
     end
 
@@ -85,8 +85,8 @@ describe Lhm::AtomicSwitcher do
       switcher.run
 
       slave do
-        data_source_exists?(@destination).must_equal false
-        table_read(@origin.name).columns.keys.must_include 'destination'
+        value(data_source_exists?(@destination)).must_equal false
+        value(table_read(@origin.name).columns.keys).must_include 'destination'
       end
     end
   end

data/spec/integration/chunk_insert_spec.rb

@@ -22,7 +22,7 @@ describe Lhm::ChunkInsert do
       @instance.insert_and_return_count_of_rows_created
 
       slave do
-        count_all(@destination.name).must_equal(1)
+        value(count_all(@destination.name)).must_equal(1)
       end
     end
   end

data/spec/integration/chunker_spec.rb

@@ -15,34 +15,120 @@ describe Lhm::Chunker do
       @origin = table_create(:origin)
       @destination = table_create(:destination)
       @migration = Lhm::Migration.new(@origin, @destination)
+      @logs = StringIO.new
+      Lhm.logger = Logger.new(@logs)
+    end
+
+    def log_messages
+      @logs.string.split("\n")
     end
 
     it 'should copy 1 row from origin to destination even if the id of the single row does not start at 1' do
       execute("insert into origin set id = 1001 ")
-      printer = Lhm::Printer::Base.new
 
-      def printer.notify(*) ;end
-      def printer.end(*) [] ;end
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
+
+      slave do
+        value(count_all(@destination.name)).must_equal(1)
+      end
+
+    end
 
-      Lhm::Chunker.new(@migration, connection, {:throttler => Lhm::Throttler::Time.new(:stride => 100), :printer => printer} ).run
+    it 'should copy and ignore duplicate primary key' do
+      execute("insert into origin set id = 1001 ")
+      execute("insert into origin set id = 1002 ")
+      execute("insert into destination set id = 1002 ")
+
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
       slave do
-        count_all(@destination.name).must_equal(1)
+        value(count_all(@destination.name)).must_equal(2)
+      end
+    end
+
+    it 'should copy and ignore duplicate composite primary key' do
+      origin = table_create(:composite_primary_key)
+      destination = table_create(:composite_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into composite_primary_key set id = 1001, shop_id = 1")
+      execute("insert into composite_primary_key set id = 1002, shop_id = 1")
+      execute("insert into composite_primary_key_dest set id = 1002, shop_id = 1")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      slave do
+        value(count_all(destination.name)).must_equal(2)
+      end
+    end
+
+    it 'should copy and raise on unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      exception = assert_raises(Lhm::Error) do
+        Lhm::Chunker.new(migration, connection, {raise_on_warnings: true, throttler: throttler, printer: printer} ).run
       end
 
+      assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'", exception.message
+    end
+
+    it 'should copy and warn on unexpected warnings by default' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      assert_equal 2, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
+    end
+
+    it 'should log two times for two unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key set id = 1002, pk = 2")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 3")
+      execute("insert into custom_primary_key_dest set id = 1002, pk = 4")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      assert_equal 3, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
+      assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key 'index_custom_primary_key_on_id'"), log_messages
+    end
+
+    it 'should copy and warn on unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      Lhm::Chunker.new(migration, connection, {raise_on_warnings: false, throttler: throttler, printer: printer} ).run
+
+      assert_equal 2, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
     end
 
     it 'should create the modified destination, even if the source is empty' do
       execute("truncate origin ")
-      printer = Lhm::Printer::Base.new
 
-      def printer.notify(*) ;end
-      def printer.end(*) [] ;end
-
-      Lhm::Chunker.new(@migration, connection, {:throttler => Lhm::Throttler::Time.new(:stride => 100), :printer => printer} ).run
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
       slave do
-        count_all(@destination.name).must_equal(0)
+        value(count_all(@destination.name)).must_equal(0)
       end
 
     end
@@ -55,11 +141,11 @@ describe Lhm::Chunker do
       printer.expect(:end, :return_value, [])
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => Lhm::Throttler::Time.new(:stride => 100), :printer => printer }
+        @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
       slave do
-        count_all(@destination.name).must_equal(23)
+        value(count_all(@destination.name)).must_equal(23)
       end
 
       printer.verify
@@ -69,17 +155,13 @@ describe Lhm::Chunker do
     it 'should copy all the records of a table, even if the last chunk starts with the last record of it.' do
       11.times { |n| execute("insert into origin set id = '#{ n + 1 }'") }
 
-      printer = Lhm::Printer::Base.new
-
-      def printer.notify(*) ;end
-      def printer.end(*) [] ;end
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => Lhm::Throttler::Time.new(:stride => 10), :printer => printer }
+        @migration, connection, { throttler: Lhm::Throttler::Time.new(stride: 10), printer: printer }
       ).run
 
       slave do
-        count_all(@destination.name).must_equal(11)
+        value(count_all(@destination.name)).must_equal(11)
       end
 
     end
@@ -92,11 +174,11 @@ describe Lhm::Chunker do
       printer.expect(:end, :return_value, [])
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => Lhm::Throttler::SlaveLag.new(:stride => 100), :printer => printer }
+        @migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
       ).run
 
       slave do
-        count_all(@destination.name).must_equal(23)
+        value(count_all(@destination.name)).must_equal(23)
       end
 
       printer.verify
@@ -109,19 +191,19 @@ describe Lhm::Chunker do
       printer.expects(:notify).with(instance_of(Integer), instance_of(Integer)).twice
       printer.expects(:end)
 
-      throttler = Lhm::Throttler::SlaveLag.new(:stride => 10, :allowed_lag => 0)
+      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
       def throttler.max_current_slave_lag
         1
       end
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => throttler, :printer => printer }
+        @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
       assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
 
       slave do
-        count_all(@destination.name).must_equal(15)
+        value(count_all(@destination.name)).must_equal(15)
       end
     end
 
@@ -133,7 +215,7 @@ describe Lhm::Chunker do
       printer.expects(:verify)
       printer.expects(:end)
 
-      throttler = Lhm::Throttler::SlaveLag.new(:stride => 10, :allowed_lag => 0)
+      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
 
       def throttler.slave_hosts
         ['127.0.0.1']
@@ -149,14 +231,14 @@ describe Lhm::Chunker do
       end
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => throttler, :printer => printer }
+        @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
       assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
       assert_equal(0, throttler.send(:max_current_slave_lag))
 
       slave do
-        count_all(@destination.name).must_equal(15)
+        value(count_all(@destination.name)).must_equal(15)
       end
 
       printer.verify
@@ -171,14 +253,14 @@ describe Lhm::Chunker do
 
       exception = assert_raises do
         Lhm::Chunker.new(
-          @migration, connection, { :verifier => failer, :printer => printer, :throttler => Lhm::Throttler::Time.new(:stride => 100) }
+          @migration, connection, { verifier: failer, printer: printer, throttler: throttler }
         ).run
       end
 
       assert_match "Verification failed, aborting early", exception.message
 
       slave do
-        count_all(@destination.name).must_equal(0)
+        value(count_all(@destination.name)).must_equal(0)
       end
     end
   end