lhm-shopify 3.3.6 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73574a1b69152ceda698d97bda7b8ccc92e006fb8a8caedc756d9667a63e1f04
-  data.tar.gz: 1b4df5f80d8fc05d857c04fd43beb6822424574fd95dbe259f32e2847d5dfe52
+  metadata.gz: c60de7019efd80463c07434cb2312197009347dc5e7b8d238c1f3ba0355b7a9e
+  data.tar.gz: 44658f6f74b6c98739c6fd58046dd3ffa20436949f379eb9607a070afabeee8e
 SHA512:
-  metadata.gz: 5c3f86f66d4bb9ef14b001106bcf83ba91befa3cbbc3996f5e38688598b25ebf340eca23c0144bd1b17d0896ad04f978cc33b3a2c2a9bcadcdd39044930f12c1
-  data.tar.gz: e937acda8252f5ae2aa9883ef60c83409ca92602312316eec11360b440b40d8c79bf36654c2dfcdd82ce745a04f328c8482ae304059b5730bca9ee6ca2501141
+  metadata.gz: 318c3a3f051c8bfd8cb8e30bdfef2bad0c0c61fcbc7081780adb4eaba60023e94ac9d86b4f203a950ad31c304cc3bdd3affb88eb29633ef115185aab92596d55
+  data.tar.gz: 644b728df70002076404e93133dbed4f53565bec70de02ca39640a482276ad7480de18a45a06a277bd55c1d6cad906026a7454bd05e7471efd055921da51708a

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 3.4.0 (Jul 19, 2021)
+
+* Log or raise on unexpected duplicated entry warnings during INSERT IGNORE (https://github.com/Shopify/lhm/pull/100)
+
 # 3.3.6 (Jul 7, 2021)
 
 * Add lhm-shopify.rb to require lhm

data/lib/lhm/chunker.rb

@@ -20,6 +20,7 @@ module Lhm
       @connection = connection
       @chunk_finder = ChunkFinder.new(migration, connection, options)
       @options = options
+      @raise_on_warnings = options.fetch(:raise_on_warnings, false)
       @verifier = options[:verifier]
       if @throttler = options[:throttler]
         @throttler.connection = @connection if @throttler.respond_to?(:connection=)
@@ -44,6 +45,12 @@ module Lhm
         verify_can_run
 
         affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @options).insert_and_return_count_of_rows_created
+        expected_rows = top - bottom + 1
+
+        if affected_rows < expected_rows
+          raise_on_non_pk_duplicate_warning
+        end
+
         if @throttler && affected_rows > 0
           @throttler.run
         end
@@ -59,6 +66,16 @@ module Lhm
 
     private
 
+    def raise_on_non_pk_duplicate_warning
+      @connection.query("show warnings").each do |level, code, message|
+        unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
+          m = "Unexpected warning found for inserted row: #{message}"
+          Lhm.logger.warn(m)
+          raise Error.new(m) if @raise_on_warnings
+        end
+      end
+    end
+
     def bottom
       @next_to_insert
     end

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.3.6'
+  VERSION = '3.4.0'
 end

data/spec/fixtures/composite_primary_key.ddl

@@ -1,7 +1,6 @@
-
 CREATE TABLE `composite_primary_key` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,
   `shop_id` bigint(20) NOT NULL,
-  PRIMARY KEY (`shop_id`,`id`),
+  CONSTRAINT `pk_composite` PRIMARY KEY (`shop_id`,`id`),
   INDEX `index_key_id` (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/composite_primary_key_dest.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `composite_primary_key_dest` (
+  `id` bigint(20) NOT NULL AUTO_INCREMENT,
+  `shop_id` bigint(20) NOT NULL,
+  CONSTRAINT `pk_composite` PRIMARY KEY (`shop_id`,`id`),
+  INDEX `index_key_id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/fixtures/custom_primary_key_dest.ddl

@@ -0,0 +1,6 @@
+CREATE TABLE `custom_primary_key_dest` (
+  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `pk` varchar(255),
+  PRIMARY KEY (`pk`),
+  UNIQUE KEY `index_custom_primary_key_on_id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8

data/spec/integration/chunker_spec.rb

@@ -15,16 +15,18 @@ describe Lhm::Chunker do
       @origin = table_create(:origin)
       @destination = table_create(:destination)
       @migration = Lhm::Migration.new(@origin, @destination)
+      @logs = StringIO.new
+      Lhm.logger = Logger.new(@logs)
+    end
+
+    def log_messages
+      @logs.string.split("\n")
     end
 
     it 'should copy 1 row from origin to destination even if the id of the single row does not start at 1' do
       execute("insert into origin set id = 1001 ")
-      printer = Lhm::Printer::Base.new
-
-      def printer.notify(*) ;end
-      def printer.end(*) [] ;end
 
-      Lhm::Chunker.new(@migration, connection, {:throttler => Lhm::Throttler::Time.new(:stride => 100), :printer => printer} ).run
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
       slave do
         count_all(@destination.name).must_equal(1)
@@ -32,14 +34,98 @@ describe Lhm::Chunker do
 
     end
 
+    it 'should copy and ignore duplicate primary key' do
+      execute("insert into origin set id = 1001 ")
+      execute("insert into origin set id = 1002 ")
+      execute("insert into destination set id = 1002 ")
+
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
+
+      slave do
+        count_all(@destination.name).must_equal(2)
+      end
+    end
+
+    it 'should copy and ignore duplicate composite primary key' do
+      origin = table_create(:composite_primary_key)
+      destination = table_create(:composite_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into composite_primary_key set id = 1001, shop_id = 1")
+      execute("insert into composite_primary_key set id = 1002, shop_id = 1")
+      execute("insert into composite_primary_key_dest set id = 1002, shop_id = 1")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      slave do
+        count_all(destination.name).must_equal(2)
+      end
+    end
+
+    it 'should copy and raise on unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      exception = assert_raises(Lhm::Error) do
+        Lhm::Chunker.new(migration, connection, {raise_on_warnings: true, throttler: throttler, printer: printer} ).run
+      end
+
+      assert_match "Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'", exception.message
+    end
+
+    it 'should copy and warn on unexpected warnings by default' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      assert_equal 2, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
+    end
+
+    it 'should log two times for two unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key set id = 1002, pk = 2")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 3")
+      execute("insert into custom_primary_key_dest set id = 1002, pk = 4")
+
+      Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
+
+      assert_equal 3, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
+      assert log_messages[2].include?("Unexpected warning found for inserted row: Duplicate entry '1002' for key 'index_custom_primary_key_on_id'"), log_messages
+    end
+
+    it 'should copy and warn on unexpected warnings' do
+      origin = table_create(:custom_primary_key)
+      destination = table_create(:custom_primary_key_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into custom_primary_key set id = 1001, pk = 1")
+      execute("insert into custom_primary_key_dest set id = 1001, pk = 2")
+
+      Lhm::Chunker.new(migration, connection, {raise_on_warnings: false, throttler: throttler, printer: printer} ).run
+
+      assert_equal 2, log_messages.length
+      assert log_messages[1].include?("Unexpected warning found for inserted row: Duplicate entry '1001' for key 'index_custom_primary_key_on_id'"), log_messages
+    end
+
     it 'should create the modified destination, even if the source is empty' do
       execute("truncate origin ")
-      printer = Lhm::Printer::Base.new
 
-      def printer.notify(*) ;end
-      def printer.end(*) [] ;end
-
-      Lhm::Chunker.new(@migration, connection, {:throttler => Lhm::Throttler::Time.new(:stride => 100), :printer => printer} ).run
+      Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
       slave do
         count_all(@destination.name).must_equal(0)
@@ -55,7 +141,7 @@ describe Lhm::Chunker do
       printer.expect(:end, :return_value, [])
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => Lhm::Throttler::Time.new(:stride => 100), :printer => printer }
+        @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
       slave do
@@ -69,13 +155,9 @@ describe Lhm::Chunker do
     it 'should copy all the records of a table, even if the last chunk starts with the last record of it.' do
       11.times { |n| execute("insert into origin set id = '#{ n + 1 }'") }
 
-      printer = Lhm::Printer::Base.new
-
-      def printer.notify(*) ;end
-      def printer.end(*) [] ;end
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => Lhm::Throttler::Time.new(:stride => 10), :printer => printer }
+        @migration, connection, { throttler: Lhm::Throttler::Time.new(stride: 10), printer: printer }
       ).run
 
       slave do
@@ -92,7 +174,7 @@ describe Lhm::Chunker do
       printer.expect(:end, :return_value, [])
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => Lhm::Throttler::SlaveLag.new(:stride => 100), :printer => printer }
+        @migration, connection, { throttler: Lhm::Throttler::SlaveLag.new(stride: 100), printer: printer }
       ).run
 
       slave do
@@ -109,13 +191,13 @@ describe Lhm::Chunker do
       printer.expects(:notify).with(instance_of(Integer), instance_of(Integer)).twice
       printer.expects(:end)
 
-      throttler = Lhm::Throttler::SlaveLag.new(:stride => 10, :allowed_lag => 0)
+      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
       def throttler.max_current_slave_lag
         1
       end
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => throttler, :printer => printer }
+        @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
       assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
@@ -133,7 +215,7 @@ describe Lhm::Chunker do
       printer.expects(:verify)
       printer.expects(:end)
 
-      throttler = Lhm::Throttler::SlaveLag.new(:stride => 10, :allowed_lag => 0)
+      throttler = Lhm::Throttler::SlaveLag.new(stride: 10, allowed_lag: 0)
 
       def throttler.slave_hosts
         ['127.0.0.1']
@@ -149,7 +231,7 @@ describe Lhm::Chunker do
       end
 
       Lhm::Chunker.new(
-        @migration, connection, { :throttler => throttler, :printer => printer }
+        @migration, connection, { throttler: throttler, printer: printer }
       ).run
 
       assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT, throttler.timeout_seconds)
@@ -171,7 +253,7 @@ describe Lhm::Chunker do
 
       exception = assert_raises do
         Lhm::Chunker.new(
-          @migration, connection, { :verifier => failer, :printer => printer, :throttler => Lhm::Throttler::Time.new(:stride => 100) }
+          @migration, connection, { verifier: failer, printer: printer, throttler: throttler }
         ).run
       end
 

data/spec/test_helper.rb

@@ -30,3 +30,16 @@ def without_verbose(&block)
 ensure
   $VERBOSE = old_verbose
 end
+
+def printer
+  printer = Lhm::Printer::Base.new
+
+  def printer.notify(*) ;end
+  def printer.end(*) [] ;end
+
+  printer
+end
+
+def throttler
+  Lhm::Throttler::Time.new(:stride => 100)
+end

data/spec/unit/chunker_spec.rb

@@ -41,6 +41,7 @@ describe Lhm::Chunker do
       @connection.expects(:select_value).with(regexp_matches(/where id >= 8 order by id limit 1 offset 4/)).returns(21)
       @connection.expects(:update).with(regexp_matches(/between 1 and 7/)).returns(2)
       @connection.expects(:update).with(regexp_matches(/between 8 and 10/)).returns(2)
+      @connection.expects(:query).twice.with(regexp_matches(/show warnings/)).returns([])
 
       @chunker.run
     end
@@ -88,6 +89,8 @@ describe Lhm::Chunker do
       @connection.expects(:update).with(regexp_matches(/between 6 and 8/)).returns(2)
       @connection.expects(:update).with(regexp_matches(/between 9 and 10/)).returns(2)
 
+      @connection.expects(:query).twice.with(regexp_matches(/show warnings/)).returns([])
+
       @chunker.run
     end
 
@@ -136,6 +139,7 @@ describe Lhm::Chunker do
 
       @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
       @connection.expects(:update).with(regexp_matches(/where \(foo.created_at > '2013-07-10' or foo.baz = 'quux'\) and `foo`/)).returns(1)
+      @connection.expects(:query).with(regexp_matches(/show warnings/)).returns([])
 
       def @migration.conditions
         "where foo.created_at > '2013-07-10' or foo.baz = 'quux'"
@@ -155,6 +159,7 @@ describe Lhm::Chunker do
 
       @connection.expects(:select_value).with(regexp_matches(/where id >= 1 order by id limit 1 offset 1/)).returns(2)
       @connection.expects(:update).with(regexp_matches(/inner join bar on foo.id = bar.foo_id and/)).returns(1)
+      @connection.expects(:query).with(regexp_matches(/show warnings/)).returns([])
 
       def @migration.conditions
         'inner join bar on foo.id = bar.foo_id'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 3.3.6
+  version: 3.4.0
 platform: ruby
 authors:
 - SoundCloud
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-08 00:00:00.000000000 Z
+date: 2021-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable
@@ -173,7 +173,9 @@ files:
 - spec/README.md
 - spec/fixtures/bigint_table.ddl
 - spec/fixtures/composite_primary_key.ddl
+- spec/fixtures/composite_primary_key_dest.ddl
 - spec/fixtures/custom_primary_key.ddl
+- spec/fixtures/custom_primary_key_dest.ddl
 - spec/fixtures/destination.ddl
 - spec/fixtures/lines.ddl
 - spec/fixtures/origin.ddl