lhm-teak 3.6.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b84f8808e479b9588bcd03e14dd67ef885b058d2abec860e174a947545ff2af0
+  data.tar.gz: 644b6c7e3565d091a8a24d05d35d4077c63846c62761ff637ebc754cd1be846a
+SHA512:
+  metadata.gz: 046cf2c7d36a0ca272bcb0b0130bfd115cc3723610b6b4f9e8d74fe9757a96c3ab3a2e0be779810a6d7a2894ca660e518ffb407b21371de60e8f31ec45113c61
+  data.tar.gz: e60d5ad405e2a7cbc2f9d057a13ad7fc85bd89d0f6472685a5d72a33cc19b8878688e94a8f2968466028db4947cadcdaaa7fb415f442f7140d803d373ee3e21e

data/.github/workflows/test.yml

@@ -0,0 +1,43 @@
+name: Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - ruby-version: "2.4"
+            activerecord-version: "5.2"
+          - ruby-version: "2.5"
+            activerecord-version: "6.0"
+          - ruby-version: "2.7.5"
+            activerecord-version: "6.1"
+          - ruby-version: "3.0"
+            activerecord-version: "7.0.0.alpha2"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{matrix.ruby-version}}
+          bundler-cache: false
+      - name: Install Ruby packages
+        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle install
+      - name: Install Ubuntu packages
+        run: sudo apt-get update && sudo apt-get install numactl libaio-dev libmysqlclient-dev
+      - name: Setup MySQL and ProxySQL (docker-compose)
+        run: docker-compose up -d # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
+      - name: Wait until DBs are alive
+        run: ./scripts/helpers/wait-for-dbs.sh
+        timeout-minutes: 2
+      - name: Run tests
+        run: BUNDLE_GEMFILE="${GITHUB_WORKSPACE}/gemfiles/activerecord_${{ matrix.activerecord-version }}.gemfile" bundle exec rake specs

data/.gitignore

@@ -0,0 +1,12 @@
+*.gem
+.bundle
+pkg/*
+.rvmrc
+.ruby-version
+.ruby-gemset
+bin/rake
+gemfiles/vendor
+omg.ponies
+*~
+coverage
+.idea/

data/.rubocop.yml

@@ -0,0 +1,183 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2015-01-12 14:49:41 +0100 using RuboCop version 0.26.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+AllCops:
+  Exclude:
+    - 'gemfiles/**/*'
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 131
+
+# Offense count: 3
+Metrics/CyclomaticComplexity:
+  Max: 19
+
+# Offense count: 77
+# Configuration parameters: AllowURI, URISchemes.
+Metrics/LineLength:
+  Max: 197
+
+# Offense count: 11
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 27
+
+# Offense count: 3
+Metrics/PerceivedComplexity:
+  Max: 15
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/AndOr:
+  Enabled: false
+
+# Offense count: 13
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/BarePercentLiterals:
+  Enabled: false
+
+# Offense count: 11
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/BracesAroundHashParameters:
+  Enabled: false
+
+# Offense count: 5
+Style/ClassVars:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredMethods.
+Style/CollectionMethods:
+  Enabled: false
+
+# Offense count: 24
+Style/Documentation:
+  Enabled: false
+
+# Offense count: 28
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/DotPosition:
+  Enabled: false
+
+# Offense count: 4
+Style/DoubleNegation:
+  Enabled: false
+
+# Offense count: 1
+Style/EachWithObject:
+  Enabled: false
+
+# Offense count: 1
+# Configuration parameters: Exclude.
+Style/FileName:
+  Enabled: false
+
+# Offense count: 11
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/FormatString:
+  Enabled: false
+
+# Offense count: 10
+# Configuration parameters: AllowedVariables.
+Style/GlobalVars:
+  Enabled: false
+
+# Offense count: 9
+# Configuration parameters: MinBodyLength.
+Style/GuardClause:
+  Enabled: false
+
+# Offense count: 88
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/HashSyntax:
+  Enabled: false
+
+# Offense count: 3
+# Configuration parameters: MaxLineLength.
+Style/IfUnlessModifier:
+  Enabled: false
+
+# Offense count: 14
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/IndentHash:
+  Enabled: false
+
+# Offense count: 2
+Style/ModuleFunction:
+  Enabled: false
+
+# Offense count: 2
+Style/MultilineBlockChain:
+  Enabled: false
+
+# Offense count: 4
+# Cop supports --auto-correct.
+Style/NumericLiterals:
+  MinDigits: 8
+
+# Offense count: 14
+# Cop supports --auto-correct.
+# Configuration parameters: PreferredDelimiters.
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+# Offense count: 3
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/RaiseArgs:
+  Enabled: false
+
+# Offense count: 1
+Style/RescueModifier:
+  Enabled: false
+
+# Offense count: 1
+Style/SelfAssignment:
+  Enabled: false
+
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/SignalException:
+  EnforcedStyle: only_raise
+  Enabled: true
+
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SupportedStyles.
+Style/SpaceInsideHashLiteralBraces:
+  SupportedStyles: space
+
+# Offense count: 2
+# Cop supports --auto-correct.
+Style/SpecialGlobalVars:
+  Enabled: false
+
+# Offense count: 5
+# Cop supports --auto-correct.
+# Configuration parameters: ExactNameMatch, AllowPredicates, AllowDSLWriters, Whitelist.
+Style/TrivialAccessors:
+  Enabled: false
+
+# Offense count: 15
+# Cop supports --auto-correct.
+Style/UnneededPercentQ:
+  Enabled: false
+
+# Offense count: 3
+# Cop supports --auto-correct.
+Style/WordArray:
+  MinSize: 2
+
+# Offense count: 7
+Metrics/AbcSize:
+  Max: 31

data/.travis.yml

@@ -0,0 +1,21 @@
+language: ruby
+before_script:
+  - "mysql -e 'create database lhm;'"
+rvm:
+  - 2.0.0
+  - 2.1
+  - 2.2
+sudo: false
+gemfile:
+  - gemfiles/ar-2.3_mysql.gemfile
+  - gemfiles/ar-3.2_mysql.gemfile
+  - gemfiles/ar-3.2_mysql2.gemfile
+  - gemfiles/ar-4.0_mysql2.gemfile
+  - gemfiles/ar-4.1_mysql2.gemfile
+  - gemfiles/ar-4.2_mysql2.gemfile
+matrix:
+  exclude:
+    - rvm: 2.2
+      gemfile: gemfiles/ar-3.2_mysql.gemfile
+    - rvm: 2.2
+      gemfile: gemfiles/ar-2.3_mysql.gemfile

data/Appraisals

@@ -0,0 +1,24 @@
+# Oldest supported Rails version
+appraise "activerecord-5.2" do
+  gem "activerecord", "5.2.6"
+
+# This gemfile will be ran against ruby 2.4 and some gems might require 2.4+. The following list are the gems
+# and their respective versions that work with ruby <= 2.4
+  gem "simplecov", "0.18.5"
+  gem "docile", "1.3.5"
+end
+
+# First conflicted version
+appraise "activerecord-6.0" do
+  gem "activerecord", "6.0.0"
+end
+
+# Second conflicted version
+appraise "activerecord-6.1" do
+  gem "activerecord", "6.1.0"
+end
+
+# Latest version at the moment
+appraise "activerecord-7.0.0.alpha2" do
+  gem "activerecord", "7.0.0.alpha2"
+end

data/CHANGELOG.md

@@ -0,0 +1,254 @@
+# 3.5.5 (Jan, 2022)
+* Fix error where from Config shadowing which would cause LHM to abort on reconnect (https://github.com/Shopify/lhm/pull/128)
+
+# 3.5.4 (Dec, 2021)
+* Refactored the way options are handled internally. Code is now much clearer to understand
+* Removed optional connection_options from `Lhm.setup` and `Lhm.connection`
+* Option `reconnect_with_consistent_host` will now be provided with `options` for `Lhm.change_table`
+
+# 3.5.3 (Dec, 2021)
+* Adds ProxySQL comments at the end of query to accommodate for internal tool's requirements
+
+# 3.5.2 (Dec, 2021)
+* Fixed error on undefined connection, when calling `Lhm.connection` without calling `Lhm.setup` first
+* Changed `Lhm.connection.connection` to `lhm.connection.ar_connection` for increased clarity and readability
+
+# 3.5.1 (Dec , 2021)
+* Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/112)
+* Slave lag throttler now supports ActiveRecord > 6.0
+* [Dev] Add `Appraisals` to test against multiple version
+
+# 3.5.0 (Dec , 2021)
+* Duplicate of 3.4.2 (unfortunate mistake)
+
+# 3.4.2 (Sept, 2021)
+* Fixed Chunker's undefined name error (https://github.com/Shopify/lhm/pull/110)
+
+# 3.4.1 (Sep 22, 2021)
+
+* Add better logging to the LHM components (https://github.com/Shopify/lhm/pull/108)
+
+# 3.4.0 (Jul 19, 2021)
+
+* Log or raise on unexpected duplicated entry warnings during INSERT IGNORE (https://github.com/Shopify/lhm/pull/100)
+
+# 3.3.6 (Jul 7, 2021)
+
+* Add lhm-shopify.rb to require lhm
+
+# 3.3.5 (Jul 5, 2021)
+
+* Add comment and collate copying to rename_column
+* Publish to rubygems
+
+# 3.3.4 (Feb 9, 2021)
+
+* Run migrations inline in local/CI environment
+
+# 3.3.3 (Nov 20, 2020)
+
+* Add test for tables with composite primary keys.
+* Add test for migrating to a composite primary key.
+* Tests updated to work on MacOS Catalina
+* LHM will now print exceptions to @printer if @printer responds to :exception
+* New ThreadsRunning throttler uses MySQL Performance Schema to decide whether to throttle
+
+# 3.3.2 (not fully released)
+
+* Catch _even_ more MySQL errors by default with SqlRetry.
+
+# 3.3.1 (Nov 8, 2019)
+
+* Ensure that :retriable configuration is correctly passed to all SqlRetry
+  instances.
+* Retry `Chunker#upper_id` and `options[:verifier]` on MySQL failure.
+* Catch more MySQL errors by default with SqlRetry.
+
+# 3.3.0 (Oct 21, 2019)
+
+* Add a :verifier key to the options hash, with a default implementation which aborts the LHM if the triggers are removed.
+
+# 3.2.5 (Jun 24, 2019)
+
+* Tighten dependency on retriable gem and remove workarounds for old version
+
+# 3.2.4 (Oct 16, 2018)
+
+* Retry `Cleanup::Current` just like we retry all the other DDLs.
+
+# 3.2.3 (Oct 16, 2018)
+
+* Add ActiveRecord::QueryTimedout exception class to be retried on "Timeout waiting for a response from the last query" message.
+
+# 3.2.2 (Oct 11, 2018)
+
+* Try to take a higher lock_wait_timeout value than others  (https://github.com/Shopify/lhm/pull/60)
+
+# 3.2.1 (Oct 11, 2018)
+
+* Retry on `MySQL::Error::Timeout` (https://github.com/Shopify/lhm/pull/57)
+* Retry 20 times by default (https://github.com/Shopify/lhm/pull/58)
+
+# 3.2.0 (Sep 4, 2018)
+
+* Fix Slave lag throttler database config (https://github.com/Shopify/lhm/pull/55)
+* Loosen dependency on retriable gem (https://github.com/Shopify/lhm/pull/54)
+* Overhaul retries for deadlocks, wait timeouts on Chunker, Entangler, and AtomicSwitcher (https://github.com/Shopify/lhm/pull/51)
+
+# 3.1.1
+
+* Cleanup tables between tests (https://github.com/Shopify/lhm/pull/48)
+* Ensure all table names are less than 64 characters (https://github.com/Shopify/lhm/pull/49)
+
+# 3.1.0
+
+* Unify Entangler and AtomicSwitcher retry interface (https://github.com/Shopify/lhm/pull/39)
+* Remove scripts replaced by dbdeployer (https://github.com/Shopify/lhm/pull/40)
+* Rename lhmn_ tables to lhma_ to avoid IBP stalls (https://github.com/Shopify/lhm/pull/41)
+
+# 3.0.0
+
+* Drop support for throttle and stride options. Use `throttler`, instead:
+```
+Lhm.change_table :users, throttler: [:time_throttler, {stride: x}] do
+end
+```
+* #118 - Truncate long trigger names. (@sj26)
+* #114 - Update chunker requirements (@bjk-soundcloud)
+* #98 - Add slave lag throttler. (@camilo, @jasonhl)
+* #92 - Fix check for table requirement before starting a lhm.(@hannestyden)
+* #93 - Makes the atomic switcher retry on metadata locks (@camilo)
+* #63 - Sets the LHM's session lock wait timeout variables (@camilo)
+* #75 - Remove DataMapper and ActiveRecord 2.x support (@camilo)
+
+# 2.2.0 (Jan 16, 2015)
+
+* #84 - Require index names to be strings or symbols (Thibaut)
+* #39 - Adding the ability to rename columns (erikogan)
+* #67 - Allow for optional time filter on .cleanup (joelr)
+
+# 2.1.0 (July 31, 2014)
+
+* #48 - Add percentage output for migrations (@arthurnn)
+* #60 - Quote table names (@spickermann)
+* #59 - Escape table name in select_limit and select_start methods (@stevehodgkiss)
+* #57 - Ensure chunking 'where' clause handled separately (@rentalcustard)
+* #54 - Chunker handle stride changes (@rentalcustard)
+* #52 - Implement ability to control timeout and stride from Throttler (@edmundsalvacion)
+* #51 - Ensure Lhm.cleanup removes temporary triggers (@edmundsalvacion)
+* #46 - Allow custom throttler (@arthurnn)
+
+# 2.0.0 (July 10, 2013)
+
+* #44 - Conditional migrations (@durran)
+
+# 1.3.0 (May 28, 2013)
+
+* Add Lhm.cleanup method for removing copy tables, thanks @bogdan
+* Limit copy table names to 64 characters, thanks @charliesome
+
+# 1.2.0 (February 22, 2013)
+
+* Added DataMapper support, no API changes for current users. Refer to the
+  README for information.
+* Documentation updates. Thanks @tiegz and @vinbarnes.
+
+# 1.1.0 (April 29, 2012)
+
+* Add option to specify custom index name
+* Add mysql2 compatibility
+* Add AtomicSwitcher
+
+# 1.0.3 (February 23, 2012)
+
+* Improve change_column
+
+# 1.0.2 (February 17, 2012)
+
+* closes https://github.com/soundcloud/large-hadron-migrator/issues/11
+  this critical bug could cause data loss. table parser was replaced with
+  an implementation that reads directly from information_schema.
+
+# 1.0.1 (February 09, 2012)
+
+* released to rubygems
+
+# 1.0.0 (February 09, 2012)
+
+* added change_column
+* final 1.0 release
+
+# 1.0.0.rc8 (February 09, 2012)
+
+* removed spec binaries from gem bins
+
+# 1.0.0.rc7 (January 31, 2012)
+
+* added SqlHelper.annotation into the middle of trigger statements. this
+  is for the benefit of the killer script which should not kill trigger
+  statements.
+
+# 1.0.0.rc6 (January 30, 2012)
+
+* added --confirm to kill script; fixes to kill script
+
+# 1.0.0.rc5 (January 30, 2012)
+
+* moved scripts into bin, renamed, added to gem binaries
+
+# 1.0.0.rc4 (January 29, 2012)
+
+* added '-- lhm' to the end of statements for more visibility
+
+# 1.0.0.rc3 (January 19, 2012)
+
+* Speedup migrations for tables with large minimum id
+* Add a bit yard documentation
+* Fix issues with index creation on reserved column names
+* Improve error handling
+* Add tests for replication
+* Rename public API method from `hadron_change_table` to `change_table`
+* Add tests for ActiveRecord 2.3 and 3.1 compatibility
+
+# 1.0.0.rc2 (January 18, 2012)
+
+* Speedup migrations for tables with large ids
+* Fix conversion of milliseconds to seconds
+* Fix handling of sql errors
+* Add helper to create unique index
+* Allow index creation on prefix of column
+* Quote column names on index creation
+* Remove ambiguous method signature
+* Documentation fix
+* 1.8.7 compatibility
+
+# 1.0.0.rc1 (January 15, 2012)
+
+* rewrite.
+
+# 0.2.1 (November 26, 2011)
+
+* Include changelog in gem
+
+# 0.2.0 (November 26, 2011)
+
+* Add Ruby 1.8 compatibility
+* Setup travis continuous integration
+* Fix record lose issue
+* Fix and speed up specs
+
+# 0.1.4
+
+* Merged [Pullrequest #9](https://github.com/soundcloud/large-hadron-migrator/pull/9)
+
+# 0.1.3
+
+* code cleanup
+* Merged [Pullrequest #8](https://github.com/soundcloud/large-hadron-migrator/pull/8)
+* Merged [Pullrequest #7](https://github.com/soundcloud/large-hadron-migrator/pull/7)
+* Merged [Pullrequest #4](https://github.com/soundcloud/large-hadron-migrator/pull/4)
+* Merged [Pullrequest #1](https://github.com/soundcloud/large-hadron-migrator/pull/1)
+
+# 0.1.2
+
+* Initial Release

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shopify_lhm.gemspec
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,67 @@
+PATH
+  remote: .
+  specs:
+    lhm-teak (3.6.0)
+      retriable (>= 3.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activesupport (6.1.4.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    after_do (0.4.0)
+    appraisal (2.4.1)
+      bundler
+      rake
+      thor (>= 0.14.0)
+    byebug (11.1.3)
+    concurrent-ruby (1.1.9)
+    docile (1.4.0)
+    i18n (1.8.11)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    mocha (1.13.0)
+    mysql2 (0.5.3)
+    rake (13.0.6)
+    retriable (3.1.2)
+    simplecov (0.21.2)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.3)
+    thor (1.1.0)
+    toxiproxy (2.0.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.5.1)
+
+PLATFORMS
+  arm64-darwin-21
+  x86_64-darwin-20
+  x86_64-linux
+
+DEPENDENCIES
+  activerecord
+  after_do
+  appraisal
+  byebug
+  lhm-teak!
+  minitest
+  mocha
+  mysql2
+  rake
+  simplecov
+  toxiproxy
+
+BUNDLED WITH
+   2.2.22

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011 - 2013, SoundCloud, Rany Keddo, Tobias Bielohlawek, Tobias Schmidt
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+- Neither the name of the SoundCloud nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+