lhm-shopify 3.5.1 → 3.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b5fbf9905c14cd938075e3f6b930de0b52cdbc7c048f8ed572c654824681d99
-  data.tar.gz: c950dd9650c4e04412a91a9d21b6bbff3b3f092ecbbce1474a77518716737d62
+  metadata.gz: c3893b743c675e62933e58a4f56c0ac2dba1b1081afabc779dc1d899406e7c42
+  data.tar.gz: 569175938b8069e0036f881400b9427180283ec61c29ca5d509a7090a7f8a9ba
 SHA512:
-  metadata.gz: e92c7cc7cefa85e18a9c5b3d17966574324c8fe8e067f4ac4203241d18e8dae836c82554f92685e90978af26451422852c654fed8f0b855032dee0ac3d0f130d
-  data.tar.gz: 8755a2814d7d92d015d35ae3636993e39a66b5d27d1980a002eec96a2e33012350310d5f903b5f3b225d6b4bf60295841ee4e35919b927914a0cc700683d357b
+  metadata.gz: ba3ebf953f97cd793c52cb83fee77dd4436877ca01a51ee6d718c7937916e391a32c659b9f403b62f65cf0a326dc755e6221e3311760738c9c6fb442746e1617
+  data.tar.gz: 2e617329a895d00a2541539526e941fc94dfda8285083d46d4179587a31786cbbb1062bd8dbdfffcc4d046801944ac96aa700c32618a4db9f4346345e9105c7c

data/CHANGELOG.md

@@ -1,3 +1,29 @@
+# 3.5.5 (Jan, 2022)
+* Fix error where from Config shadowing which would cause LHM to abort on reconnect (https://github.com/Shopify/lhm/pull/128)
+
+# 3.5.4 (Dec, 2021)
+* Refactored the way options are handled internally. Code is now much clearer to understand
+* Removed optional connection_options from `Lhm.setup` and `Lhm.connection`
+* Option `reconnect_with_consistent_host` will now be provided with `options` for `Lhm.change_table`
+
+# 3.5.3 (Dec, 2021)
+* Adds ProxySQL comments at the end of query to accommodate for internal tool's requirements
+
+# 3.5.2 (Dec, 2021)
+* Fixed error on undefined connection, when calling `Lhm.connection` without calling `Lhm.setup` first
+* Changed `Lhm.connection.connection` to `lhm.connection.ar_connection` for increased clarity and readability
+
+# 3.5.1 (Dec , 2021)
+* Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/112)
+* Slave lag throttler now supports ActiveRecord > 6.0
+* [Dev] Add `Appraisals` to test against multiple version
+
+# 3.5.0 (Dec , 2021)
+* Duplicate of 3.4.2 (unfortunate mistake)
+
+# 3.4.2 (Sept, 2021)
+* Fixed Chunker's undefined name error (https://github.com/Shopify/lhm/pull/110)
+
 # 3.4.1 (Sep 22, 2021)
 
 * Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/108)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (3.5.1)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/README.md

@@ -113,7 +113,7 @@ tables must be cleaned up.
 LHM can recover from connection loss. However, when used in conjunction with ProxySQL, there are multiple ways that
 connection loss could induce data loss (if triggered by a failover). Therefore  it will perform additional checks to
 ensure that the MySQL host stays consistent across the schema migrations if the feature is enabled. 
-This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL. 
+This is done by tagging every query with `/*maintenance:lhm*/`, which will be recognized by ProxySQL.
 However, to get this feature working, a new ProxySQL query rule must be added.
 ```cnf
 {
@@ -145,9 +145,11 @@ forwarded to the right target.
 ```
 
 Once these changes are added to the ProxySQL configuration (either through `.cnf` or dynamically through the admin interface), 
-the feature can be enabled. This is done by adding this flag when doing the initial setup:
+the feature can be enabled. This is done by adding this flag when providing options to the migration:
 ```ruby
- Lhm.setup(connection, options: {reconnect_with_consistent_host: true})
+ Lhm.change_table(..., options: {reconnect_with_consistent_host: true}) do |t|
+  ...
+end
 ```
 **Note**: This feature is disabled by default
 
@@ -301,6 +303,15 @@ COV=1 bundle exec rake unit && bundle exec rake integration
 open coverage/index.html
 ```
 
+### Merging for a new version
+When creating a PR for a new version, make sure that th version has been bumped in `lib/lhm/version.rb`. Then run the following code snippet to ensure the everything is consistent, otherwise 
+the gem will not publish.
+```bash
+bundle install
+bundle update
+bundle exec appraisals install
+```
+
 ### Docker Compose
 The integration tests rely on a replication configuration for MySQL which is being proxied by an instance of ProxySQL.
 It is important that every container is running to execute the integration test suite.

data/Rakefile

@@ -16,16 +16,16 @@ Rake::TestTask.new('integration') do |t|
   t.libs << 'spec'
   t.test_files = FileList['spec/integration/**/*_spec.rb']
   t.verbose = true
-  end
+end
 
 Rake::TestTask.new('dev') do |t|
   t.libs << 'lib'
   t.libs << 'spec'
-  t.test_files = FileList[
-    'spec/test_helper.rb',
-    'spec/integration/lhm_spec.rb'
-  #  Add file to test individually
-  ]
+
+  files = FileList.new('spec/test_helper.rb')
+  files.add(ENV["SINGLE_TEST"]) if ENV["SINGLE_TEST"]
+  t.test_files = files
+
   t.verbose = true
 end
 

data/dev.yml

@@ -5,7 +5,7 @@ up:
           or: [mysql@5.7]
           conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
       - wget
-  - ruby: 2.7.0
+  - ruby: 2.7.5
   - bundler
   - custom:
       name: Get Appraisal gems
@@ -31,7 +31,7 @@ commands:
       if [[ $# -eq 0 ]]; then
         bundle exec rake unit && bundle exec rake integration
       else
-        bundle exec rake dev TEST="$@"
+        SINGLE_TEST="$@" bundle exec rake dev
       fi
   appraisals: bundle exec appraisal rake specs
   cov: rm -rf coverage; COV=1 bundle exec rake unit && bundle exec rake integration; open coverage/index.html
@@ -40,3 +40,6 @@ commands:
     run: docker-compose logs -f
   clear:
     run: docker-compose rm -v -s -f && docker-compose up -d && ./scripts/helpers/wait-for-dbs.sh
+  pre-publish:
+    # Ensures all Gemfile.lock are sync with the new version in `lhm/version.rb` and runs appraisals
+    run: bundle install && bundle exec appraisal install && bundle exec appraisal rake specs

data/gemfiles/activerecord_5.2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.1)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_6.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.1)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.1)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.0.0.alpha2.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (3.5.1)
+    lhm-shopify (3.5.5)
       retriable (>= 3.0.0)
 
 GEM

data/lib/lhm/atomic_switcher.rb

@@ -16,12 +16,13 @@ module Lhm
 
     attr_reader :connection
 
-    def initialize(migration, connection = nil, options={})
+    LOG_PREFIX = "AtomicSwitcher"
+
+    def initialize(migration, connection = nil)
       @migration = migration
       @connection = connection
       @origin = migration.origin
       @destination = migration.destination
-      @retry_options = options[:retriable] || {}
     end
 
     def atomic_switch
@@ -39,7 +40,7 @@ module Lhm
     private
 
     def execute
-      @connection.execute(atomic_switch, should_retry: true, retry_options: @retry_options)
+      @connection.execute(atomic_switch, should_retry: true, log_prefix: LOG_PREFIX)
     end
   end
 end

data/lib/lhm/chunk_insert.rb

@@ -3,16 +3,19 @@ require 'lhm/proxysql_helper'
 
 module Lhm
   class ChunkInsert
-    def initialize(migration, connection, lowest, highest, options = {})
+
+    LOG_PREFIX = "ChunkInsert"
+
+    def initialize(migration, connection, lowest, highest, retry_options = {})
       @migration = migration
       @connection = connection
       @lowest = lowest
       @highest = highest
-      @retry_options = options[:retriable] || {}
+      @retry_options = retry_options
     end
 
     def insert_and_return_count_of_rows_created
-      @connection.update(sql, should_retry: true, retry_options: @retry_options)
+      @connection.update(sql, should_retry: true, log_prefix: LOG_PREFIX)
     end
 
     def sql

data/lib/lhm/chunker.rb

@@ -13,6 +13,8 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "Chunker"
+
     # Copy from origin to destination in chunks of size `stride`.
     # Use the `throttler` class to sleep between each stride.
     def initialize(migration, connection = nil, options = {})
@@ -31,9 +33,7 @@ module Lhm
       @retry_options = options[:retriable] || {}
       @retry_helper = SqlRetry.new(
         @connection,
-        options: {
-          log_prefix: "Chunker"
-        }.merge!(@retry_options)
+        retry_options: @retry_options
       )
     end
 
@@ -79,7 +79,7 @@ module Lhm
     private
 
     def raise_on_non_pk_duplicate_warning
-      @connection.execute("show warnings", should_retry: true, retry_options: @retry_options).each do |level, code, message|
+      @connection.execute("show warnings", should_retry: true, log_prefix: LOG_PREFIX).each do |level, code, message|
         unless message.match?(/Duplicate entry .+ for key 'PRIMARY'/)
           m = "Unexpected warning found for inserted row: #{message}"
           Lhm.logger.warn(m)
@@ -94,14 +94,14 @@ module Lhm
 
     def verify_can_run
       return unless @verifier
-      @retry_helper.with_retries(@retry_options) do |retriable_connection|
+      @retry_helper.with_retries(log_prefix: LOG_PREFIX) do |retriable_connection|
         raise "Verification failed, aborting early" if !@verifier.call(retriable_connection)
       end
     end
 
     def upper_id(next_id, stride)
       sql = "select id from `#{ @migration.origin_name }` where id >= #{ next_id } order by id limit 1 offset #{ stride - 1}"
-      top = @connection.select_value(sql, should_retry: true, retry_options: @retry_options)
+      top = @connection.select_value(sql, should_retry: true, log_prefix: LOG_PREFIX)
 
       [top ? top.to_i : @limit, @limit].min
     end

data/lib/lhm/cleanup/current.rb

@@ -4,6 +4,9 @@ require 'lhm/sql_retry'
 module Lhm
   module Cleanup
     class Current
+
+      LOG_PREFIX = "Current"
+
       def initialize(run, origin_table_name, connection, options={})
         @run = run
         @table_name = TableName.new(origin_table_name)
@@ -54,7 +57,7 @@ module Lhm
 
       def execute_ddls
         ddls.each do |ddl|
-            @connection.execute(ddl, should_retry: true, retry_options: @retry_config)
+            @connection.execute(ddl, should_retry: true, log_prefix: LOG_PREFIX)
         end
         Lhm.logger.info("Dropped triggers on #{@lhm_triggers_for_origin.join(', ')}")
         Lhm.logger.info("Dropped tables #{@lhm_triggers_for_origin.join(', ')}")

data/lib/lhm/connection.rb

@@ -1,20 +1,23 @@
 require 'delegate'
+require 'forwardable'
 require 'lhm/sql_retry'
 
 module Lhm
-  class Connection < SimpleDelegator
-
     # Lhm::Connection inherits from SingleDelegator. It will forward any unknown method calls to the ActiveRecord
     # connection.
-    alias connection __getobj__
-    alias connection= __setobj__
+  class Connection < SimpleDelegator
+    extend Forwardable
 
-    def initialize(connection:, default_log_prefix: nil, options: {}, retry_config: {})
-      @default_log_prefix = default_log_prefix
-      @retry_options = retry_config || default_retry_config
+    # Will delegate the following function to @sql_retry object, while leaving them accessible from the Lhm::Connection
+    # object
+    def_delegators :@sql_retry, :reconnect_with_consistent_host, :reconnect_with_consistent_host=, :retry_config=
+
+    alias ar_connection __getobj__
+
+    def initialize(connection:, options: {})
       @sql_retry = Lhm::SqlRetry.new(
         connection,
-        options: retry_config,
+        retry_options: options[:retriable] || {},
         reconnect_with_consistent_host: options[:reconnect_with_consistent_host] || false
       )
 
@@ -22,46 +25,51 @@ module Lhm
       super(connection)
     end
 
-    def options=(options)
-      # If any other flags are added. Add the "processing" here
-      @sql_retry.reconnect_with_consistent_host = options[:reconnect_with_consistent_host] || false
+    def ar_connection=(connection)
+      raise Lhm::Error.new("Lhm::Connection requires an active record connection to operate") if connection.nil?
+
+      @sql_retry.connection = connection
+      # Sets connection as the delegated object
+      __setobj__(connection)
     end
 
-    def execute(query, should_retry: false, retry_options: {})
+    # ActiveRecord::Base overridden methods to incorporate custom retry logic
+    # All other methods will be delegated
+    def execute(query, should_retry: false, log_prefix: nil)
       if should_retry
-        exec_with_retries(:execute, query, retry_options)
+        exec_with_retries(:execute, query, log_prefix)
       else
         exec(:execute, query)
       end
     end
 
-    def update(query, should_retry: false, retry_options: {})
+    def update(query, should_retry: false, log_prefix: nil)
       if should_retry
-        exec_with_retries(:update, query, retry_options)
+        exec_with_retries(:update, query, log_prefix)
       else
         exec(:update, query)
       end
     end
 
-    def select_value(query, should_retry: false, retry_options: {})
+    def select_value(query, should_retry: false, log_prefix: nil)
       if should_retry
-        exec_with_retries(:select_value, query, retry_options)
+        exec_with_retries(:select_value, query, log_prefix)
       else
         exec(:select_value, query)
       end
     end
 
-    def select_values(query, should_retry: false, retry_options: {})
+    def select_values(query, should_retry: false, log_prefix: nil)
       if should_retry
-        exec_with_retries(:select_values, query, retry_options)
+        exec_with_retries(:select_values, query, log_prefix)
       else
         exec(:select_values, query)
       end
     end
 
-    def select_one(query, should_retry: false, retry_options: {})
+    def select_one(query, should_retry: false, log_prefix: nil)
       if should_retry
-        exec_with_retries(:select_one, query, retry_options)
+        exec_with_retries(:select_one, query, log_prefix)
       else
         exec(:select_one, query)
       end
@@ -70,12 +78,12 @@ module Lhm
     private
 
     def exec(method, sql)
-      connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
+      ar_connection.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
     end
 
-    def exec_with_retries(method, sql, retry_options = {})
-      retry_options[:log_prefix] ||= file
-      @sql_retry.with_retries(retry_options) do |conn|
+    def exec_with_retries(method, sql, log_prefix=nil)
+      effective_log_prefix = log_prefix || file
+      @sql_retry.with_retries(log_prefix: effective_log_prefix) do |conn|
         conn.public_send(method, Lhm::ProxySQLHelper.tagged(sql))
       end
     end

data/lib/lhm/entangler.rb

@@ -13,14 +13,15 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "Entangler"
+
     # Creates entanglement between two tables. All creates, updates and deletes
     # to origin will be repeated on the destination table.
-    def initialize(migration, connection = nil, options = {})
+    def initialize(migration, connection = nil)
       @intersection = migration.intersection
       @origin = migration.origin
       @destination = migration.destination
       @connection = connection
-      @retry_options = options[:retriable] || {}
     end
 
     def entangle
@@ -86,14 +87,14 @@ module Lhm
 
     def before
       entangle.each do |stmt|
-        @connection.execute(stmt, should_retry: true, retry_options: @retry_options)
+        @connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
       end
       Lhm.logger.info("Created triggers on #{@origin.name}")
     end
 
     def after
       untangle.each do |stmt|
-        @connection.execute(stmt, should_retry: true, retry_options: @retry_options)
+        @connection.execute(stmt, should_retry: true, log_prefix: LOG_PREFIX)
       end
       Lhm.logger.info("Dropped triggers on #{@origin.name}")
     end

data/lib/lhm/invoker.rb

@@ -16,8 +16,8 @@ module Lhm
   class Invoker
     include SqlHelper
     LOCK_WAIT_TIMEOUT_DELTA = 10
-    INNODB_LOCK_WAIT_TIMEOUT_MAX=1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
-    LOCK_WAIT_TIMEOUT_MAX=31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
+    INNODB_LOCK_WAIT_TIMEOUT_MAX = 1073741824.freeze # https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_lock_wait_timeout
+    LOCK_WAIT_TIMEOUT_MAX = 31536000.freeze # https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html
 
     attr_reader :migrator, :connection
 
@@ -49,7 +49,7 @@ module Lhm
       normalize_options(options)
       set_session_lock_wait_timeouts
       migration = @migrator.run
-      entangler = Entangler.new(migration, @connection, options)
+      entangler = Entangler.new(migration, @connection)
 
       entangler.run do
         options[:verifier] ||= Proc.new { |conn| triggers_still_exist?(conn, entangler) }
@@ -90,6 +90,8 @@ module Lhm
         options[:throttler] = Lhm.throttler
       end
 
+      Lhm.connection.retry_config = options[:retriable] || {}
+
     rescue => e
       Lhm.logger.error "LHM run failed with exception=#{e.class} message=#{e.message}"
       raise

data/lib/lhm/locked_switcher.rb

@@ -22,6 +22,8 @@ module Lhm
 
     attr_reader :connection
 
+    LOG_PREFIX = "LockedSwitcher"
+
     def initialize(migration, connection = nil)
       @migration = migration
       @connection = connection

data/lib/lhm/proxysql_helper.rb

@@ -4,7 +4,7 @@ module Lhm
     ANNOTATION = "/*maintenance:lhm*/"
 
     def tagged(sql)
-      "#{ANNOTATION}#{sql}"
+      "#{sql} #{ANNOTATION}"
     end
   end
 end