lhm-shopify 4.2.1 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dec22d00a0a019706b1a6d62e85e416caf210aa2d0b4f77bb3ccf562f1db8fb2
-  data.tar.gz: 181acf59e61b100dc3320b6674a7cd13a2a1dcf767c7259a88a58dbf4b826bf5
+  metadata.gz: f0c79dc83baf664e8cd4ad2e11f8e72b3927edbb2af0beb5626961339e7f75c7
+  data.tar.gz: 1a58b01ab3ab768c37f497e24329d0e4e3d85dc64af7b1a338c5447b742f4a4b
 SHA512:
-  metadata.gz: 8c197de7b7658cdb9c31dd91adde7fdaf54505bc7e910812d8c80edf447f6266630c08a80d773391071c518f72b9527280e1ece7e2bdc00bc60d5a8c22f294ed
-  data.tar.gz: 6f1f7da4e1bd2e61fb8921eb9e724408d01ee576d98eb8ba393c2c2dada846368ec2a187e5c61d77f3691290edc815addd0e93acb8e1fa065c206564243ef798
+  metadata.gz: e349310577754cebfe7fa4d067727946c19fb0b291f1f2305e87d26a7d30371fd37c80bcdb58039936c90c75618157904f3141722e5f3f83c9b0e05afb76e447
+  data.tar.gz: 63fe5f4a016b32ab3d299e5cd9368f226a575bd725040931b8dd28db7e8a35cef7fbd78f0ec2713108dd508a974c7b783734a02e4d724a3b40e00afb7502693e

data/.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+registries:
+  ruby-shopify:
+    type: rubygems-server
+    url: https://pkgs.shopify.io/basic/gems/ruby
+    username: ${{secrets.RUBYGEMS_SERVER_PKGS_SHOPIFY_IO_USERNAME}}
+    password: ${{secrets.RUBYGEMS_SERVER_PKGS_SHOPIFY_IO_PASSWORD}}
+  github-com:
+    type: git
+    url: https://github.com
+    username: ${{secrets.DEPENDENCIES_GITHUB_USER}}
+    password: ${{secrets.DEPENDENCIES_GITHUB_TOKEN}}
+updates:
+  - package-ecosystem: bundler
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 100
+    insecure-external-code-execution: allow
+    registries: "*"

data/.github/workflows/test.yml

@@ -13,9 +13,10 @@ jobs:
 
     strategy:
       fail-fast: false
+      max-parallel: 8
       matrix:
-        activerecord: ["6.1", "7.0", "7.1"]
-        ruby: ["3.0", "3.1", "3.2", "head"]
+        activerecord: ["6.1", "7.0", "7.1", "head"]
+        ruby: ["3.1", "3.2", "3.3", "head"]
         mysql: ["5.7", "8.0"]
         adapter: ["mysql2", "trilogy"]
         exclude:
@@ -23,6 +24,8 @@ jobs:
             ruby: head
           - activerecord: 7.0
             ruby: head
+          - activerecord: head
+            ruby: 3.0
 
     env:
       BUNDLE_GEMFILE: "${{ github.workspace }}/gemfiles/activerecord_${{ matrix.activerecord }}.gemfile"
@@ -40,7 +43,7 @@ jobs:
 
       - name: Setup MySQL and ProxySQL (docker-compose)
         # Might have to change to docker compose up -d (i.e. Compose V2) when the Ubuntu image changes the docker-compose version
-        run: docker-compose -f docker-compose-mysql-${{ matrix.mysql }}.yml up -d
+        run: docker compose -f docker-compose-mysql-${{ matrix.mysql }}.yml up -d
 
       - name: Wait until DBs are alive
         run: ./scripts/helpers/wait-for-dbs.sh

data/.gitignore

@@ -2,7 +2,6 @@
 .bundle
 pkg/*
 .rvmrc
-.ruby-version
 .ruby-gemset
 bin/rake
 gemfiles/vendor

data/.ruby-version

@@ -0,0 +1 @@
+3.2.2

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Unreleased
 
+# 4.3.0 (Aug, 2024)
+* Drop support for Ruby 3.0, as it reached its EOL
+* Add support for next Rails version
+
+# 4.2.3 (Jul, 2024)
+* Fix check for warnings against PKs with line breaks
+
+# 4.2.2 (Jun, 2024)
+* Avoid using the INSTANT algorithm.
+
 # 4.2.1 (Mar, 2024)
 * Retry more errors when using Trilogy.
 

data/Gemfile.lock

@@ -1,19 +1,19 @@
 PATH
   remote: .
   specs:
-    lhm-shopify (4.2.1)
+    lhm-shopify (4.3.0)
       retriable (>= 3.0.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (7.1.1)
-      activesupport (= 7.1.1)
-    activerecord (7.1.1)
-      activemodel (= 7.1.1)
-      activesupport (= 7.1.1)
+    activemodel (7.1.3.4)
+      activesupport (= 7.1.3.4)
+    activerecord (7.1.3.4)
+      activemodel (= 7.1.3.4)
+      activesupport (= 7.1.3.4)
       timeout (>= 0.4.0)
-    activesupport (7.1.1)
+    activesupport (7.1.3.4)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.0.2)
@@ -28,22 +28,21 @@ GEM
       bundler
       rake
       thor (>= 0.14.0)
-    base64 (0.1.1)
-    bigdecimal (3.1.4)
+    base64 (0.2.0)
+    bigdecimal (3.1.8)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.3.3)
     connection_pool (2.4.1)
-    docile (1.4.0)
-    drb (2.1.1)
-      ruby2_keywords
-    i18n (1.14.1)
+    docile (1.4.1)
+    drb (2.2.1)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
-    minitest (5.22.2)
-    mocha (2.1.0)
+    minitest (5.24.1)
+    mocha (2.4.5)
       ruby2_keywords (>= 0.0.5)
-    mutex_m (0.1.2)
-    mysql2 (0.5.5)
-    rake (13.0.6)
+    mutex_m (0.2.0)
+    mysql2 (0.5.6)
+    rake (13.2.1)
     retriable (3.1.2)
     ruby2_keywords (0.0.5)
     simplecov (0.22.0)
@@ -52,10 +51,10 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    thor (1.2.2)
-    timeout (0.4.0)
+    thor (1.3.1)
+    timeout (0.4.1)
     toxiproxy (2.0.2)
-    trilogy (2.6.0)
+    trilogy (2.8.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
 

data/dev.yml

@@ -1,11 +1,9 @@
 name: lhm
 up:
-  - homebrew:
-      - mysql-client@5.7:
-          or: [mysql@5.7]
-          conflicts: [shopify/shopify/mysql-client, mysql-connector-c, mysql, mysql-client]
+  - packages:
+      - mysql-client@5.7
       - wget
-  - ruby: 3.2.2
+  - ruby
   - bundler
   - podman
   - custom:

data/gemfiles/activerecord_6.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.1)
+    lhm-shopify (4.3.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.0.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.1)
+    lhm-shopify (4.3.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_7.1.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    lhm-shopify (4.2.1)
+    lhm-shopify (4.3.0)
       retriable (>= 3.0.0)
 
 GEM

data/gemfiles/activerecord_head.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "activerecord", git: "https://github.com/rails/rails.git", branch: "main"
+
+gemspec path: "../"

data/lib/lhm/chunker.rb

@@ -23,7 +23,6 @@ module Lhm
       @chunk_finder = ChunkFinder.new(migration, connection, options)
       @options = options
       @raise_on_warnings = options.fetch(:raise_on_warnings, false)
-      @pk_duplicate_warning_regexp ||= /Duplicate entry .+ for key '(#{@migration.destination_name}\.)?PRIMARY'/
       @verifier = options[:verifier]
       if @throttler = options[:throttler]
         @throttler.connection = @connection if @throttler.respond_to?(:connection=)
@@ -81,7 +80,7 @@ module Lhm
 
     def raise_on_non_pk_duplicate_warning
       @connection.select_all("SHOW WARNINGS", should_retry: true, log_prefix: LOG_PREFIX).each do |row|
-        next if row["Message"].match?(@pk_duplicate_warning_regexp)
+        next if row["Message"].start_with?("Duplicate entry") && row["Message"].match?(/for key '(#{@migration.destination_name}\.)?PRIMARY'\z/)
 
         m = "Unexpected warning found for inserted row: #{row["Message"]}"
         Lhm.logger.warn(m)

data/lib/lhm/migrator.rb

@@ -53,7 +53,7 @@ module Lhm
     # @param [String] name Name of the column to add
     # @param [String] definition Valid SQL column definition
     def add_column(name, definition)
-      ddl('alter table `%s` add column `%s` %s' % [@name, name, definition])
+      ddl('alter table `%s` add column `%s` %s, ALGORITHM=INPLACE' % [@name, name, definition])
     end
 
     # Change an existing column to a new definition
@@ -94,7 +94,7 @@ module Lhm
       definition += " COMMENT #{@connection.quote(col[:comment])}" if col[:comment]
       definition += " COLLATE #{@connection.quote(col[:collate])}" if col[:collate]
 
-      ddl('alter table `%s` change column `%s` `%s` %s' % [@name, old, nu, definition])
+      ddl('alter table `%s` change column `%s` `%s` %s, ALGORITHM=INPLACE' % [@name, old, nu, definition])
       @renames[old.to_s] = nu.to_s
     end
 
@@ -108,7 +108,7 @@ module Lhm
     #
     # @param [String] name Name of the column to delete
     def remove_column(name)
-      ddl('alter table `%s` drop `%s`' % [@name, name])
+      ddl('alter table `%s` drop `%s`, ALGORITHM=INPLACE' % [@name, name])
     end
 
     # Add an index to a table

data/lib/lhm/sql_helper.rb

@@ -65,7 +65,7 @@ module Lhm
     end
 
     def struct_key(struct, key)
-      keys = if struct.is_a? Hash
+      keys = if struct.respond_to?(:keys)
                struct.keys
              else
                struct.members

data/lib/lhm/version.rb

@@ -2,5 +2,5 @@
 # Schmidt
 
 module Lhm
-  VERSION = '4.2.1'
+  VERSION = '4.3.0'
 end

data/spec/fixtures/composite_primary_key_with_varchar_columns.ddl

@@ -0,0 +1,10 @@
+CREATE TABLE `composite_primary_key_with_varchar_columns` (
+  `id` bigint NOT NULL AUTO_INCREMENT,
+  `shop_id` bigint NOT NULL,
+  `owner_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  `owner_id` bigint NOT NULL,
+  `namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  `key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  PRIMARY KEY (`shop_id`,`owner_type`,`owner_id`,`namespace`,`key`),
+  UNIQUE KEY `id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC

data/spec/fixtures/composite_primary_key_with_varchar_columns_dest.ddl

@@ -0,0 +1,10 @@
+CREATE TABLE `composite_primary_key_with_varchar_columns_dest` (
+  `id` bigint NOT NULL AUTO_INCREMENT,
+  `shop_id` bigint NOT NULL,
+  `owner_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  `owner_id` bigint NOT NULL,
+  `namespace` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  `key` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL DEFAULT '',
+  PRIMARY KEY (`shop_id`,`owner_type`,`owner_id`,`namespace`,`key`),
+  UNIQUE KEY `id` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci ROW_FORMAT=DYNAMIC

data/spec/integration/chunker_spec.rb

@@ -61,6 +61,46 @@ describe Lhm::Chunker do
       end
     end
 
+    it 'should copy and ignore duplicate composite primary key with line breaks' do
+      origin = table_create(:composite_primary_key_with_varchar_columns)
+      destination = table_create(:composite_primary_key_with_varchar_columns_dest)
+      migration = Lhm::Migration.new(origin, destination)
+
+      execute("insert into composite_primary_key_with_varchar_columns set id = 1001, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = '
+  23
+
+  23
+', `key` = '
+  14
+
+  1
+'")
+      execute("insert into composite_primary_key_with_varchar_columns set id = 1002, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = '
+  23
+
+  22
+', `key` = '
+  14
+
+  1
+'")
+      execute("insert into composite_primary_key_with_varchar_columns_dest set id = 1002, shop_id = 1, owner_type = 'Product', owner_id = 1, namespace = '
+  23
+
+  22
+', `key` = '
+  14
+
+  1
+'")
+
+      Lhm::Chunker.new(migration, connection, {raise_on_warning: true, throttler: throttler, printer: printer} ).run
+
+      replica do
+        value(count_all(destination.name)).must_equal(2)
+      end
+    end
+
     it 'should copy and raise on unexpected warnings' do
       origin = table_create(:custom_primary_key)
       destination = table_create(:custom_primary_key_dest)

data/spec/integration/cleanup_spec.rb

@@ -2,6 +2,7 @@
 # Schmidt
 
 require File.expand_path(File.dirname(__FILE__)) + '/integration_helper'
+require 'ostruct'
 
 describe Lhm, 'cleanup' do
   include IntegrationHelper

data/spec/integration/lhm_spec.rb

@@ -328,8 +328,7 @@ describe Lhm do
           :collate => collation,
         })
 
-        result = select_one('SELECT login from users')
-        result = result['login'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT login from users')
         value(result).must_equal('a user')
       end
     end
@@ -353,8 +352,7 @@ describe Lhm do
           :collate => collation,
         })
 
-        result = select_one('SELECT `fnord` from users')
-        result = result['fnord'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `fnord` from users')
         value(result).must_equal('Superfriends')
       end
     end
@@ -380,8 +378,7 @@ describe Lhm do
              :collate => 'utf8mb4_unicode_ci',
            })
 
-        result = select_one('SELECT `user_name` from users')
-        result = result['user_name'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `user_name` from users')
         value(result).must_equal('a user')
       end
     end
@@ -410,8 +407,7 @@ describe Lhm do
            :collate => nil,
          })
 
-        result = select_one('SELECT `ref` from users')
-        result = result['ref'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `ref` from users')
         value(result).must_equal(10)
       end
     end
@@ -437,8 +433,7 @@ describe Lhm do
            :collate => collation,
          })
 
-        result = select_one('SELECT `fnord` from users')
-        result = result['fnord'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `fnord` from users')
         assert_nil(result)
       end
     end
@@ -462,8 +457,7 @@ describe Lhm do
           :collate => collation,
         })
 
-        result = select_one('SELECT `user_name` from users')
-        result = result['user_name'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `user_name` from users')
         value(result).must_equal('a user')
       end
     end
@@ -489,8 +483,7 @@ describe Lhm do
           :collate => collation,
         })
 
-        result = select_one('SELECT `user_name` from users')
-        result = result['user_name'] if result.respond_to?(:has_key?)
+        result = select_value('SELECT `user_name` from users')
         value(result).must_equal('a user')
       end
     end

data/spec/unit/migrator_spec.rb

@@ -97,7 +97,7 @@ describe Lhm::Migrator do
       @creator.add_column('logins', 'INT(12)')
 
       value(@creator.statements).must_equal([
-        'alter table `lhmn_alt` add column `logins` INT(12)'
+        'alter table `lhmn_alt` add column `logins` INT(12), ALGORITHM=INPLACE'
       ])
     end
 
@@ -105,7 +105,7 @@ describe Lhm::Migrator do
       @creator.remove_column('logins')
 
       value(@creator.statements).must_equal([
-        'alter table `lhmn_alt` drop `logins`'
+        'alter table `lhmn_alt` drop `logins`, ALGORITHM=INPLACE'
       ])
     end
 
@@ -151,10 +151,10 @@ describe Lhm::Migrator do
       value(@creator.statements.length).must_equal(2)
 
       value(@creator.statements[0])
-        .must_equal('alter table `lhmn_alt` add column `first` VARCHAR(64)')
+        .must_equal('alter table `lhmn_alt` add column `first` VARCHAR(64), ALGORITHM=INPLACE')
 
       value(@creator.statements[1])
-        .must_equal('alter table `lhmn_alt` add column `last` VARCHAR(64)')
+        .must_equal('alter table `lhmn_alt` add column `last` VARCHAR(64), ALGORITHM=INPLACE')
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lhm-shopify
 version: !ruby/object:Gem::Version
-  version: 4.2.1
+  version: 4.3.0
 platform: ruby
 authors:
 - SoundCloud
@@ -9,10 +9,10 @@ authors:
 - Rany Keddo
 - Tobias Bielohlawek
 - Tobias Schmidt
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-03-11 00:00:00.000000000 Z
+date: 2024-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: retriable
@@ -190,9 +190,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ".rubocop.yml"
+- ".ruby-version"
 - Appraisals
 - CHANGELOG.md
 - Gemfile
@@ -210,6 +212,7 @@ files:
 - gemfiles/activerecord_7.0.gemfile.lock
 - gemfiles/activerecord_7.1.gemfile
 - gemfiles/activerecord_7.1.gemfile.lock
+- gemfiles/activerecord_head.gemfile
 - lhm.gemspec
 - lib/lhm-shopify.rb
 - lib/lhm.rb
@@ -252,6 +255,8 @@ files:
 - spec/fixtures/bigint_table.ddl
 - spec/fixtures/composite_primary_key.ddl
 - spec/fixtures/composite_primary_key_dest.ddl
+- spec/fixtures/composite_primary_key_with_varchar_columns.ddl
+- spec/fixtures/composite_primary_key_with_varchar_columns_dest.ddl
 - spec/fixtures/custom_primary_key.ddl
 - spec/fixtures/custom_primary_key_dest.ddl
 - spec/fixtures/destination.ddl
@@ -305,7 +310,7 @@ licenses:
 - BSD-3-Clause
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -320,8 +325,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.6
-signing_key: 
+rubygems_version: 3.5.16
+signing_key:
 specification_version: 4
 summary: online schema changer for mysql
 test_files: []