lhm-shopify 3.4.0 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c60de7019efd80463c07434cb2312197009347dc5e7b8d238c1f3ba0355b7a9e
-  data.tar.gz: 44658f6f74b6c98739c6fd58046dd3ffa20436949f379eb9607a070afabeee8e
+  metadata.gz: ef0c285f7d63755816429aa2795220be251c0c93c4506f8255f7e547c074d574
+  data.tar.gz: 5de29d0b924375634c061a459837f0ea6a12d5ccf58226032ef25f708ef15628
 SHA512:
-  metadata.gz: 318c3a3f051c8bfd8cb8e30bdfef2bad0c0c61fcbc7081780adb4eaba60023e94ac9d86b4f203a950ad31c304cc3bdd3affb88eb29633ef115185aab92596d55
-  data.tar.gz: 644b728df70002076404e93133dbed4f53565bec70de02ca39640a482276ad7480de18a45a06a277bd55c1d6cad906026a7454bd05e7471efd055921da51708a
+  metadata.gz: 2d24b1d78c6fdd67129ee91a01d043ccefb1f2e8c4b8322553f1c6cbfed6237105e58eb6ddab0e143869b813682ea6b3c25e5a40e66199f76adc0cc99a97050b
+  data.tar.gz: 327b3a438ac2fb1b0763396094baed303a2ecb9a37558295b8b516b089bcca73534102351281883679263f7e69647d1129941df4970d3abae004faf93d91db62

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 3.4.1 (Sep 22, 2021)
+
+* Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/108)
+
 # 3.4.0 (Jul 19, 2021)
 
 * Log or raise on unexpected duplicated entry warnings during INSERT IGNORE (https://github.com/Shopify/lhm/pull/100)

data/lib/lhm/chunker.rb

@@ -37,6 +37,8 @@ module Lhm
     end
 
     def execute
+      @start_time = Time.now
+      
       return if @chunk_finder.table_empty?
       @next_to_insert = @start
       while @next_to_insert <= @limit || (@start == @limit)
@@ -47,6 +49,8 @@ module Lhm
         affected_rows = ChunkInsert.new(@migration, @connection, bottom, top, @options).insert_and_return_count_of_rows_created
         expected_rows = top - bottom + 1
 
+        rate_limited_log_progress
+
         if affected_rows < expected_rows
           raise_on_non_pk_duplicate_warning
         end
@@ -54,8 +58,10 @@ module Lhm
         if @throttler && affected_rows > 0
           @throttler.run
         end
-        @printer.notify(bottom, @limit)
+        
         @next_to_insert = top + 1
+        @printer.notify(bottom, @limit)
+
         break if @start == @limit
       end
       @printer.end
@@ -100,5 +106,14 @@ module Lhm
       return if @chunk_finder.table_empty?
       @chunk_finder.validate
     end
+
+    # Only log the chunker progress every 5 minutes instead of every iteration
+    def rate_limited_log_progress
+      current_time = Time.now
+      if current_time - @start_time > (5 * 60)
+        Lhm.logger.info("Inserted #{affected_rows} rows into the destination table from #{bottom} to #{top}")
+        @start_time = current_time
+      end 
+    end
   end
 end

data/lib/lhm/cleanup/current.rb

@@ -63,11 +63,12 @@ module Lhm
             retriable_connection.execute(ddl)
           end
         end
+        Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
+        Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")
       end
 
       def report_ddls
-        puts "The following DDLs would be executed:"
-        ddls.each { |ddl| puts ddl }
+        Lhm.logger.info("The following DDLs would be executed: #{ddls}")
       end
     end
   end

data/lib/lhm/entangler.rb

@@ -94,6 +94,7 @@ module Lhm
           retriable_connection.execute(stmt)
         end
       end
+      Lhm.logger.info("Created triggers on #{@origin.name}")
     end
 
     def after
@@ -102,6 +103,7 @@ module Lhm
           retriable_connection.execute(stmt)
         end
       end
+      Lhm.logger.info("Dropped triggers on #{@origin.name}")
     end
 
     def revert

data/lib/lhm/migrator.rb

@@ -214,6 +214,8 @@ module Lhm
       replacement = %{CREATE TABLE `#{ @origin.destination_name }`}
       stmt = @origin.ddl.gsub(original, replacement)
       @connection.execute(tagged(stmt))
+
+      Lhm.logger.info("Created destination table #{@origin.destination_name}")
     end
 
     def destination_read

data/lib/lhm/printer.rb

@@ -12,26 +12,30 @@ module Lhm
       end
     end
 
-    class Percentage < Base
+    class Percentage
       def initialize
-        super
         @max_length = 0
       end
 
       def notify(lowest, highest)
         return if !highest || highest == 0
+
+        # The argument lowest represents the next_to_insert row id, and highest represents the 
+        # maximum id upto which chunker has to copy the data. 
+        # If all the rows are inserted upto highest, then lowest passed here from chunker was 
+        # highest + 1, which leads to the printer printing the progress > 100%.
+        return if lowest >= highest
+        
         message = "%.2f%% (#{lowest}/#{highest}) complete" % (lowest.to_f / highest * 100.0)
         write(message)
       end
 
       def end
         write('100% complete')
-        @output.write "\n"
       end
 
       def exception(e)
-        write("failed: #{e}")
-        @output.write "\n"
+        Lhm.logger.error("failed: #{e}")
       end
 
       private
@@ -42,7 +46,7 @@ module Lhm
           extra = 0
         end
 
-        @output.write "\r#{message}" + (' ' * extra)
+        Lhm.logger.info(message)
       end
     end
 

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '3.4.0'
+  VERSION = '3.4.1'
 end

data/lib/lhm.rb

@@ -114,17 +114,21 @@ module Lhm
       triggers.each do |trigger|
         connection.execute("drop trigger if exists #{trigger}")
       end
+      logger.info("Dropped triggers #{triggers.join(', ')}")
+
       tables.each do |table|
         connection.execute("drop table if exists #{table}")
       end
+      logger.info("Dropped tables #{tables.join(', ')}")
+
       true
     elsif tables.empty? && triggers.empty?
-      puts 'Everything is clean. Nothing to do.'
+      logger.info('Everything is clean. Nothing to do.')
       true
     else
-      puts "Would drop LHM backup tables: #{tables.join(', ')}."
-      puts "Would drop LHM triggers: #{triggers.join(', ')}."
-      puts 'Run with Lhm.cleanup(true) to drop all LHM triggers and tables, or Lhm.cleanup_current_run(true, table_name) to clean up a specific LHM.'
+      logger.info("Would drop LHM backup tables: #{tables.join(', ')}.")
+      logger.info("Would drop LHM triggers: #{triggers.join(', ')}.")
+      logger.info('Run with Lhm.cleanup(true) to drop all LHM triggers and tables, or Lhm.cleanup_current_run(true, table_name) to clean up a specific LHM.')
       false
     end
   end

data/spec/integration/atomic_switcher_spec.rb

@@ -58,7 +58,7 @@ describe Lhm::AtomicSwitcher do
       switcher.send :define_singleton_method, :atomic_switch do
         'SELECT * FROM nonexistent'
       end
-      -> { switcher.run }.must_raise(ActiveRecord::StatementInvalid)
+      value(-> { switcher.run }).must_raise(ActiveRecord::StatementInvalid)
     end
 
     it "should raise when destination doesn't exist" do
@@ -75,8 +75,8 @@ describe Lhm::AtomicSwitcher do
       switcher.run
 
       slave do
-        data_source_exists?(@origin).must_equal true
-        table_read(@migration.archive_name).columns.keys.must_include 'origin'
+        value(data_source_exists?(@origin)).must_equal true
+        value(table_read(@migration.archive_name).columns.keys).must_include 'origin'
       end
     end
 
@@ -85,8 +85,8 @@ describe Lhm::AtomicSwitcher do
       switcher.run
 
       slave do
-        data_source_exists?(@destination).must_equal false
-        table_read(@origin.name).columns.keys.must_include 'destination'
+        value(data_source_exists?(@destination)).must_equal false
+        value(table_read(@origin.name).columns.keys).must_include 'destination'
       end
     end
   end

data/spec/integration/chunk_insert_spec.rb

@@ -22,7 +22,7 @@ describe Lhm::ChunkInsert do
       @instance.insert_and_return_count_of_rows_created
 
       slave do
-        count_all(@destination.name).must_equal(1)
+        value(count_all(@destination.name)).must_equal(1)
       end
     end
   end

data/spec/integration/chunker_spec.rb

@@ -29,7 +29,7 @@ describe Lhm::Chunker do
       Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
       slave do
-        count_all(@destination.name).must_equal(1)
+        value(count_all(@destination.name)).must_equal(1)
       end
 
     end
@@ -42,7 +42,7 @@ describe Lhm::Chunker do
       Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
       slave do
-        count_all(@destination.name).must_equal(2)
+        value(count_all(@destination.name)).must_equal(2)
       end
     end
 
@@ -58,7 +58,7 @@ describe Lhm::Chunker do
       Lhm::Chunker.new(migration, connection, {throttler: throttler, printer: printer} ).run
 
       slave do
-        count_all(destination.name).must_equal(2)
+        value(count_all(destination.name)).must_equal(2)
       end
     end
 
@@ -128,7 +128,7 @@ describe Lhm::Chunker do
       Lhm::Chunker.new(@migration, connection, {throttler: throttler, printer: printer} ).run
 
       slave do
-        count_all(@destination.name).must_equal(0)
+        value(count_all(@destination.name)).must_equal(0)
       end
 
     end
@@ -145,7 +145,7 @@ describe Lhm::Chunker do
       ).run
 
       slave do
-        count_all(@destination.name).must_equal(23)
+        value(count_all(@destination.name)).must_equal(23)
       end
 
       printer.verify
@@ -161,7 +161,7 @@ describe Lhm::Chunker do
       ).run
 
       slave do
-        count_all(@destination.name).must_equal(11)
+        value(count_all(@destination.name)).must_equal(11)
       end
 
     end
@@ -178,7 +178,7 @@ describe Lhm::Chunker do
       ).run
 
       slave do
-        count_all(@destination.name).must_equal(23)
+        value(count_all(@destination.name)).must_equal(23)
       end
 
       printer.verify
@@ -203,7 +203,7 @@ describe Lhm::Chunker do
       assert_equal(Lhm::Throttler::SlaveLag::INITIAL_TIMEOUT * 2 * 2, throttler.timeout_seconds)
 
       slave do
-        count_all(@destination.name).must_equal(15)
+        value(count_all(@destination.name)).must_equal(15)
       end
     end
 
@@ -238,7 +238,7 @@ describe Lhm::Chunker do
       assert_equal(0, throttler.send(:max_current_slave_lag))
 
       slave do
-        count_all(@destination.name).must_equal(15)
+        value(count_all(@destination.name)).must_equal(15)
       end
 
       printer.verify
@@ -260,7 +260,7 @@ describe Lhm::Chunker do
       assert_match "Verification failed, aborting early", exception.message
 
       slave do
-        count_all(@destination.name).must_equal(0)
+        value(count_all(@destination.name)).must_equal(0)
       end
     end
   end

data/spec/integration/cleanup_spec.rb

@@ -31,12 +31,13 @@ describe Lhm, 'cleanup' do
 
     describe 'cleanup' do
       it 'should show temporary tables' do
-        output = capture_stdout do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
           Lhm.cleanup
         end
-        output.must_include('Would drop LHM backup tables')
-        output.must_match(/lhma_[0-9_]*_users/)
-        output.must_match(/lhma_[0-9_]*_permissions/)
+        value(output).must_include('Would drop LHM backup tables')
+        value(output).must_match(/lhma_[0-9_]*_users/)
+        value(output).must_match(/lhma_[0-9_]*_permissions/)
       end
 
       it 'should show temporary tables within range' do
@@ -48,12 +49,13 @@ describe Lhm, 'cleanup' do
         table_name2 = Lhm::Migration.new(table2, nil, nil, {}, Time.now - 172800).archive_name
         table_rename(:permissions, table_name2)
 
-        output = capture_stdout do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
           Lhm.cleanup false, { :until => Time.now - 86400 }
         end
-        output.must_include('Would drop LHM backup tables')
-        output.must_match(/lhma_[0-9_]*_users/)
-        output.must_match(/lhma_[0-9_]*_permissions/)
+        value(output).must_include('Would drop LHM backup tables')
+        value(output).must_match(/lhma_[0-9_]*_users/)
+        value(output).must_match(/lhma_[0-9_]*_permissions/)
       end
 
       it 'should exclude temporary tables outside range' do
@@ -65,30 +67,40 @@ describe Lhm, 'cleanup' do
         table_name2 = Lhm::Migration.new(table2, nil, nil, {}, Time.now).archive_name
         table_rename(:permissions, table_name2)
 
-        output = capture_stdout do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
           Lhm.cleanup false, { :until => Time.now - 172800 }
         end
-        output.must_include('Would drop LHM backup tables')
-        output.wont_match(/lhma_[0-9_]*_users/)
-        output.wont_match(/lhma_[0-9_]*_permissions/)
+        value(output).must_include('Would drop LHM backup tables')
+        value(output).wont_match(/lhma_[0-9_]*_users/)
+        value(output).wont_match(/lhma_[0-9_]*_permissions/)
       end
 
       it 'should show temporary triggers' do
-        output = capture_stdout do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
           Lhm.cleanup
         end
-        output.must_include('Would drop LHM triggers')
-        output.must_include('lhmt_ins_users')
-        output.must_include('lhmt_del_users')
-        output.must_include('lhmt_upd_users')
-        output.must_include('lhmt_ins_permissions')
-        output.must_include('lhmt_del_permissions')
-        output.must_include('lhmt_upd_permissions')
+        value(output).must_include('Would drop LHM triggers')
+        value(output).must_include('lhmt_ins_users')
+        value(output).must_include('lhmt_del_users')
+        value(output).must_include('lhmt_upd_users')
+        value(output).must_include('lhmt_ins_permissions')
+        value(output).must_include('lhmt_del_permissions')
+        value(output).must_include('lhmt_upd_permissions')
       end
 
       it 'should delete temporary tables' do
-        Lhm.cleanup(true).must_equal(true)
-        Lhm.cleanup.must_equal(true)
+        value(Lhm.cleanup(true)).must_equal(true)
+        value(Lhm.cleanup).must_equal(true)
+      end
+
+      it 'outputs deleted tables and triggers' do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
+          Lhm.cleanup(true)
+        end
+        value(output).must_include('Dropped triggers lhmt_ins_users, lhmt_upd_users, lhmt_del_users, lhmt_ins_permissions, lhmt_upd_permissions, lhmt_del_permissions')
       end
     end
 
@@ -96,27 +108,26 @@ describe Lhm, 'cleanup' do
       it 'should show lhmn table for the specified table only' do
         table_create(:permissions)
         table_rename(:permissions, 'lhmn_permissions')
-        output = capture_stdout do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
           Lhm.cleanup_current_run(false, 'permissions')
-        end.split("\n")
-
-        assert_equal "The following DDLs would be executed:", output[0]
-        assert_equal "drop trigger if exists lhmt_ins_permissions", output[1]
-        assert_equal "drop trigger if exists lhmt_upd_permissions", output[2]
-        assert_equal "drop trigger if exists lhmt_del_permissions", output[3]
-        assert_match(/rename table lhmn_permissions to lhma_[0-9_]*_permissions_failed/, output[4])
-        assert_equal 5, output.length
+        end
+
+        value(output).must_include("The following DDLs would be executed:")
+        value(output).must_include("drop trigger if exists lhmt_ins_permissions")
+        value(output).must_include("drop trigger if exists lhmt_upd_permissions")
+        value(output).must_include("drop trigger if exists lhmt_del_permissions")
       end
 
       it 'should show temporary triggers for the specified table only' do
-        output = capture_stdout do
+        output = capture_stdout do |logger|
+          Lhm.logger = logger
           Lhm.cleanup_current_run(false, 'permissions')
-        end.split("\n")
-        assert_equal "The following DDLs would be executed:", output[0]
-        assert_equal "drop trigger if exists lhmt_ins_permissions", output[1]
-        assert_equal "drop trigger if exists lhmt_upd_permissions", output[2]
-        assert_equal "drop trigger if exists lhmt_del_permissions", output[3]
-        assert_equal 4, output.length
+        end
+        value(output).must_include("The following DDLs would be executed:")
+        value(output).must_include("drop trigger if exists lhmt_ins_permissions")
+        value(output).must_include("drop trigger if exists lhmt_upd_permissions")
+        value(output).must_include("drop trigger if exists lhmt_del_permissions")
       end
 
       it 'should delete temporary tables and triggers for the specified table only' do

data/spec/integration/entangler_spec.rb

@@ -26,7 +26,7 @@ describe Lhm::Entangler do
       end
 
       slave do
-        count(:destination, 'common', 'inserted').must_equal(1)
+        value(count(:destination, 'common', 'inserted')).must_equal(1)
       end
     end
 
@@ -38,7 +38,7 @@ describe Lhm::Entangler do
       end
 
       slave do
-        count(:destination, 'common', 'inserted').must_equal(0)
+        value(count(:destination, 'common', 'inserted')).must_equal(0)
       end
     end
 
@@ -49,7 +49,7 @@ describe Lhm::Entangler do
       end
 
       slave do
-        count(:destination, 'common', 'updated').must_equal(1)
+        value(count(:destination, 'common', 'updated')).must_equal(1)
       end
     end
 
@@ -59,7 +59,7 @@ describe Lhm::Entangler do
       execute("insert into origin (common) values ('inserted')")
 
       slave do
-        count(:destination, 'common', 'inserted').must_equal(0)
+        value(count(:destination, 'common', 'inserted')).must_equal(0)
       end
     end
   end

data/spec/integration/integration_helper.rb

@@ -3,6 +3,7 @@
 require 'test_helper'
 require 'yaml'
 require 'active_support'
+require 'logger'
 
 begin
   $db_config = YAML.load_file(File.expand_path(File.dirname(__FILE__)) + '/database.yml')
@@ -212,7 +213,8 @@ module IntegrationHelper
   def capture_stdout
     out = StringIO.new
     $stdout = out
-    yield
+    logger = Logger.new($stdout)
+    yield logger
     return out.string
   ensure
     $stdout = ::STDOUT