alterity 0.0.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f91f2ad32f8f398432906ae434fb824e9f94a63fc528a73a5c65c034de41ece
-  data.tar.gz: 931ed885773f0070bb68d1b354d6b79f00f7909f84614fc8c7d6f82206f41a53
+  metadata.gz: af06f20537df45280990abc9fd8c2235741b6aa22de270bbcb06684f80994d46
+  data.tar.gz: b345349594c6408775596e9189ca494671168f9b2acfecaeb9352aaa52d1b4b8
 SHA512:
-  metadata.gz: e8ecad73cb251e6aa10353eb9a6118c44e316fc85075ee8f77e8e5cf6be6d16a583f20783bfbaa7b1d7ce690b0e36f611bbf51e95bf63156588b3161adf880be
-  data.tar.gz: 39e586c77a9e9fa5d60a2e78735839457ba72a063879921d48f4ef31c488567fcf75e1fc09031564c3af0737c6e40974839a8b1cf98bdf9f9352b9bb1c75f529
+  metadata.gz: 6d012764dfd32b2002ce51f6c94b9d8cc83836e947211bb7271166c674f19ba8f61f6683fcab6a8d30cd9668103d2abc4a73477105a35004a1af662aa13612b9
+  data.tar.gz: a094fff01bebad3fee84a8391918105f5f5f0c0b0707d8b935ef05b378b27f667751b601addfe77657b14ac48c428230d9cda4e76a34ad0e206b4d961d7812b2

data/lib/alterity.rb

@@ -1,4 +1,95 @@
 # frozen_string_literal: true
 
-module Alterity
+require "rails"
+require "alterity/configuration"
+require "alterity/mysql_client_additions"
+require "alterity/railtie"
+
+class Alterity
+  class << self
+    def process_sql_query(sql, &block)
+      case sql.tr("\n", " ").strip
+      when /^alter\s+table\s+(?<table>.+?)\s+(?<updates>.+)/i
+        table = $~[:table]
+        updates = $~[:updates]
+        if updates.split(",").all? { |s| s =~ /^\s*drop\s+foreign\s+key/i } ||
+           updates.split(",").all? { |s| s =~ /^\s*add\s+constraint/i }
+          block.call
+        elsif updates =~ /drop\s+foreign\s+key/i || updates =~ /add\s+constraint/i
+          # ADD CONSTRAINT / DROP FOREIGN KEY have to go to the original table,
+          # other alterations need to got to the new table.
+          raise "[Alterity] Can't change a FK and do something else in the same query. Split it."
+        else
+          execute_alter(table, updates)
+        end
+      when /^create\s+index\s+(?<index>.+?)\s+on\s+(?<table>.+?)\s+(?<updates>.+)/i
+        execute_alter($~[:table], "ADD INDEX #{$~[:index]} #{$~[:updates]}")
+      when /^create\s+unique\s+index\s+(?<index>.+?)\s+on\s+(?<table>.+?)\s+(?<updates>.+)/i
+        execute_alter($~[:table], "ADD UNIQUE INDEX #{$~[:index]} #{$~[:updates]}")
+      when /^drop\s+index\s+(?<index>.+?)\s+on\s+(?<table>.+)/i
+        execute_alter($~[:table], "DROP INDEX #{$~[:index]}")
+      else
+        block.call
+      end
+    end
+
+    # hooks
+    def before_running_migrations
+      state.migrating = true
+      set_database_config
+      prepare_replicas_dsns_table
+    end
+
+    def after_running_migrations
+      state.migrating = false
+    end
+
+    private
+
+    def execute_alter(table, updates)
+      altered_table = table.delete("`")
+      alter_argument = %("#{updates.gsub('"', '\\"').gsub('`', '\\\`')}")
+      prepared_command = config.command.call(altered_table, alter_argument).to_s.gsub(/\n/, "\\\n")
+      puts "[Alterity] Will execute: #{prepared_command}"
+      system(prepared_command) || raise("[Alterity] Command failed")
+    end
+
+    def set_database_config
+      db_config_hash = ActiveRecord::Base.connection_db_config.configuration_hash
+      %i[host port database username password].each do |key|
+        config[key] = db_config_hash[key]
+      end
+    end
+
+    # Optional: Automatically set up table PT-OSC will monitor for replica lag.
+    def prepare_replicas_dsns_table
+      return if config.replicas_dsns_table.blank?
+
+      dsns = config.replicas_dsns.dup
+      # Automatically remove master
+      dsns.reject! { |dsn| dsn.split(",").include?("h=#{config.host}") }
+
+      database = config.replicas_dsns_database
+      table = "#{database}.#{config.replicas_dsns_table}"
+      connection = ActiveRecord::Base.connection
+      connection.execute "CREATE DATABASE IF NOT EXISTS #{database}"
+      connection.execute <<~SQL
+        CREATE TABLE IF NOT EXISTS #{table} (
+          id INT(11) NOT NULL AUTO_INCREMENT,
+          parent_id INT(11) DEFAULT NULL,
+          dsn VARCHAR(255) NOT NULL,
+          PRIMARY KEY (id)
+        ) ENGINE=InnoDB
+      SQL
+      connection.execute "TRUNCATE #{table}"
+      return if dsns.empty?
+
+      connection.execute <<~SQL
+        INSERT INTO #{table} (dsn) VALUES
+         #{dsns.map { |dsn| "('#{dsn}')" }.join(',')}
+      SQL
+    end
+  end
+
+  reset_state_and_configuration
 end

data/lib/alterity/configuration.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class Alterity
+  Configuration = Struct.new(
+    :command,
+    :host, :port, :database, :username, :password,
+    :replicas_dsns_database, :replicas_dsns_table, :replicas_dsns
+  )
+  CurrentState = Struct.new(:migrating, :disabled)
+  cattr_accessor :state
+  cattr_accessor :config
+
+  class << self
+    def reset_state_and_configuration
+      self.config = Configuration.new
+      class << config
+        def replicas(database:, table:, dsns:)
+          return ArgumentError.new("database & table must be present") if database.blank? || table.blank?
+
+          self.replicas_dsns_database = database
+          self.replicas_dsns_table = table
+          self.replicas_dsns = dsns.uniq.map do |dsn|
+            parts = dsn.split(",")
+            # automatically add default port
+            parts << "P=3306" unless parts.any? { |part| part.start_with?("P=") }
+            parts.join(",")
+          end.compact
+        end
+      end
+
+      self.state = CurrentState.new
+      load "#{__dir__}/default_configuration.rb"
+    end
+
+    def configure
+      yield config
+    end
+
+    def command=(new_command)
+      config.command = new_command
+    end
+
+    def disable
+      state.disabled = true
+      yield
+    ensure
+      state.disabled = false
+    end
+  end
+end

data/lib/alterity/default_configuration.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+Alterity.configure do |config|
+  config.command = lambda { |altered_table, alter_argument|
+    parts = ["pt-online-schema-change"]
+    parts << %(-h "#{config.host}") if config.host.present?
+    parts << %(-P "#{config.port}") if config.port.present?
+    parts << %(-u "#{config.username}") if config.username.present?
+    parts << %(--password "#{config.password.gsub('"', '\\"')}") if config.password.present?
+    parts << "--execute"
+    parts << "D=#{config.database},t=#{altered_table}"
+    parts << "--alter #{alter_argument}"
+    parts.join(" ")
+  }
+end

data/lib/alterity/mysql_client_additions.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "mysql2"
+
+class Alterity
+  module MysqlClientAdditions
+    def query(sql, options = {})
+      return super(sql, options) unless Alterity.state.migrating
+
+      Alterity.process_sql_query(sql) { super(sql, options) }
+    end
+  end
+end

data/lib/alterity/railtie.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class Alterity
+  class Railtie < Rails::Railtie
+    railtie_name :alterity
+
+    rake_tasks do
+      namespace :alterity do
+        task :intercept_table_alterations do
+          Alterity.before_running_migrations
+          Rake::Task["alterity:stop_intercepting_table_alterations"].reenable
+          ::Mysql2::Client.prepend(Alterity::MysqlClientAdditions)
+        end
+
+        task :stop_intercepting_table_alterations do
+          Rake::Task["alterity:intercept_table_alterations"].reenable
+          Alterity.after_running_migrations
+        end
+      end
+
+      unless %w[1 true].include?(ENV["DISABLE_ALTERITY"])
+        ["migrate", "migrate:up", "migrate:down", "migrate:redo", "rollback"].each do |task|
+          Rake::Task["db:#{task}"].enhance(["alterity:intercept_table_alterations"]) do
+            Rake::Task["alterity:stop_intercepting_table_alterations"].invoke
+          end
+        end
+      end
+    end
+  end
+end

data/lib/alterity/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
-module Alterity
-  VERSION = "0.0.0"
+class Alterity
+  VERSION = "1.3.0"
 end

data/spec/alterity_spec.rb

@@ -1,5 +1,59 @@
 # frozen_string_literal: true
 
 RSpec.describe Alterity do
-  # TODO
+  describe ".process_sql_query" do
+    it "executes command on table altering queries" do
+      [
+        ["ALTER TABLE `users` ADD `col` VARCHAR(255)", "`users`", "ADD `col` VARCHAR(255)"],
+        ["ALTER TABLE `users` ADD `col0` INT(11), DROP `col1`", "`users`", "ADD `col0` INT(11), DROP `col1`"],
+        ["CREATE INDEX `idx_users_on_col` ON `users` (col)", "`users`", "ADD INDEX `idx_users_on_col` (col)"],
+        ["CREATE UNIQUE INDEX `idx_users_on_col` ON `users` (col)", "`users`", "ADD UNIQUE INDEX `idx_users_on_col` (col)"],
+        ["DROP INDEX `idx_users_on_col` ON `users`", "`users`", "DROP INDEX `idx_users_on_col`"],
+        ["alter table users drop col", "users", "drop col"],
+        ["  ALTER TABLE\n   users\n   DROP col", "users", "DROP col"],
+      ].each do |(query, expected_table, expected_updates)|
+        puts query.inspect
+        expected_block = proc {}
+        expect(expected_block).not_to receive(:call)
+        expect(Alterity).to receive(:execute_alter).with(expected_table, expected_updates)
+        Alterity.process_sql_query(query, &expected_block)
+      end
+    end
+
+    it "ignores non-altering queries" do
+      [
+        "select * from users",
+        "insert into users values (1)",
+        "delete from users",
+        "begin",
+        "SHOW CREATE TABLE `users`",
+        "SHOW TABLE STATUS LIKE `users`",
+        "SHOW KEYS FROM `users`",
+        "SHOW FULL FIELDS FROM `users`",
+        "ALTER TABLE `installment_events` DROP FOREIGN KEY _fk_rails_0123456789",
+        "ALTER TABLE `installment_events` DROP FOREIGN KEY _fk_rails_0123456789, DROP FOREIGN KEY _fk_rails_9876543210",
+        "ALTER TABLE `installment_events` ADD CONSTRAINT `fk_rails_0cb5590091` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`)",
+        "ALTER TABLE `installment_events` ADD CONSTRAINT `fk_rails_0cb5590091` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`) ON DELETE CASCADE",
+      ].each do |query|
+        expected_block = proc {}
+        expect(expected_block).to receive(:call)
+        expect(Alterity).not_to receive(:execute_alter)
+        Alterity.process_sql_query(query, &expected_block)
+      end
+    end
+
+    it "raises an error if mixing FK change and other things" do
+      expected_block = proc {}
+      expect(expected_block).not_to receive(:call)
+      expect(Alterity).not_to receive(:execute_alter)
+      query = "ALTER TABLE `installment_events` ADD `col` VARCHAR(255), DROP FOREIGN KEY _fk_rails_0123456789"
+      expect do
+        Alterity.process_sql_query(query, &expected_block)
+      end.to raise_error(/FK/)
+      query = "ALTER TABLE `installment_events` ADD `col` VARCHAR(255), ADD CONSTRAINT `fk_rails_0cb5590091` FOREIGN KEY (`user_id`) REFERENCES `users` (`id`)"
+      expect do
+        Alterity.process_sql_query(query, &expected_block)
+      end.to raise_error(/FK/)
+    end
+  end
 end

data/spec/bin/custom_config.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+Alterity.configure do |config|
+  config.command = lambda { |altered_table, alter_argument|
+    string = config.to_h.slice(
+      *%i[host port username database replicas_dsns_database replicas_dsns_table replicas_dsns]
+    ).to_s
+    system("echo '#{string}' > /tmp/custom_command_result.txt")
+    system("echo '#{altered_table}' >> /tmp/custom_command_result.txt")
+    system("echo '#{alter_argument}' >> /tmp/custom_command_result.txt")
+  }
+
+  config.replicas(
+    database: "percona",
+    table: "replicas_dsns",
+    dsns: [
+      "h=host1",
+      "h=host2",
+      # we may encounter an app where the replica host is actually pointing to master;
+      # pt-osc doesn't deal well with this and will wait forever.
+      # So we're testing here that Alterity will detect that this is master (same as config.host),
+      # and will not insert it into the `replicas_dsns` table.
+      "h=#{ENV['MYSQL_HOST']}"
+    ]
+  )
+end

data/spec/bin/rails_app_migration_test.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+set -euox pipefail
+
+unset BUNDLE_GEMFILE # because we're going to create a new rails app here and use bundler
+
+sudo apt install percona-toolkit
+
+# Fixes: `Cannot connect to MySQL: Cannot get MySQL var character_set_server: DBD::mysql::db selectrow_array failed: Table 'performance_schema.session_variables' doesn't exist [for Statement "SHOW VARIABLES LIKE 'character_set_server'"] at /usr/local/Cellar/percona-toolkit/3.3.0/libexec/bin/pt-online-schema-change line 2415.`
+mysql -h $MYSQL_HOST -u $MYSQL_USERNAME -e 'set @@global.show_compatibility_56=ON'
+
+gem install rails -v $RAILS_VERSION
+
+rails new testapp \
+  --skip-action-mailer \
+  --skip-action-mailbox \
+  --skip-action-text \
+  --skip-active-job \
+  --skip-active-storage \
+  --skip-puma \
+  --skip-action-cable \
+  --skip-sprockets \
+  --skip-spring \
+  --skip-listen--skip-javascript \
+  --skip-turbolinks \
+  --skip-jbuilder--skip-test \
+  --skip-system-test \
+  --skip-bootsnap \
+  --skip-javascript \
+  --skip-webpack-install
+
+cd testapp
+
+# Sanity check:
+# echo 'gem "mysql2"' >> Gemfile
+
+echo 'gem "alterity", path: "../"' >> Gemfile
+
+bundle
+
+# Local machine test
+# echo 'development:
+#   adapter: mysql2
+#   database: alterity_test' > config/database.yml
+# bundle e rails db:drop db:create
+
+echo 'development:
+  adapter: mysql2
+  database: <%= ENV.fetch("MYSQL_DATABASE") %>
+  host: <%= ENV.fetch("MYSQL_HOST") %>
+  username: <%= ENV.fetch("MYSQL_USERNAME") %>' > config/database.yml
+
+bundle e rails g model shirt
+
+bundle e rails g migration add_color_to_shirts color:string
+
+# Test default configuration works as expected
+bundle e rails db:migrate --trace
+bundle e rails runner 'Shirt.columns.map(&:name).include?("color") || exit(1)'
+
+# Now test custom command and replication setup
+cp ../spec/bin/custom_config.rb config/initializers/alterity.rb
+
+bundle e rails g migration add_color2_to_shirts color2:string
+
+bundle e rails db:migrate --trace
+
+ruby ../spec/bin/test_custom_config_result.rb
+
+# Also testing what's in replicas_dsns, also checking that master was detected and removed.
+bundle e rails runner 'res=ActiveRecord::Base.connection.execute("select dsn from percona.replicas_dsns").to_a.flatten;p(res); res == ["h=host1,P=3306", "h=host2,P=3306"] || exit(1)'

data/spec/bin/test_custom_config_result.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+result = File.read("/tmp/custom_command_result.txt").downcase.strip
+
+expected_result = %({:host=>"127.0.0.1", :port=>nil, :username=>"root", :database=>"alterity_test", :replicas_dsns_database=>"percona", :replicas_dsns_table=>"replicas_dsns", :replicas_dsns=>["h=host1,P=3306", "h=host2,P=3306", "h=127.0.0.1,P=3306"]}
+shirts
+"ADD \\`color2\\` VARCHAR(255)").downcase.strip
+
+puts "Expected custom config result:"
+puts expected_result
+p expected_result.chars.map(&:hex)
+
+puts "Custom config result:"
+puts result
+p result.chars.map(&:hex)
+
+if result != expected_result
+  puts "=> mismatch"
+  exit(1)
+end

data/spec/spec_helper.rb

@@ -15,4 +15,8 @@ RSpec.configure do |config|
   end
 
   config.filter_run_when_matching :focus
+
+  config.before do
+    Alterity.reset_state_and_configuration
+  end
 end

metadata

@@ -1,55 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: alterity
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Chris Maximin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-28 00:00:00.000000000 Z
+date: 2021-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.1'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '8'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.1'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '8'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: mysql2
+  name: rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.3
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.5'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.3
+        version: '6.1'
 description: Execute your ActiveRecord migrations with Percona's pt-online-schema-change.
 email:
 - gems@chrismaximin.com
@@ -58,8 +46,15 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/alterity.rb
+- lib/alterity/configuration.rb
+- lib/alterity/default_configuration.rb
+- lib/alterity/mysql_client_additions.rb
+- lib/alterity/railtie.rb
 - lib/alterity/version.rb
 - spec/alterity_spec.rb
+- spec/bin/custom_config.rb
+- spec/bin/rails_app_migration_test.sh
+- spec/bin/test_custom_config_result.rb
 - spec/spec_helper.rb
 homepage: https://github.com/gumroad/alterity
 licenses:
@@ -89,4 +84,7 @@ specification_version: 4
 summary: Execute your ActiveRecord migrations with Percona's pt-online-schema-change.
 test_files:
 - spec/alterity_spec.rb
+- spec/bin/custom_config.rb
+- spec/bin/rails_app_migration_test.sh
+- spec/bin/test_custom_config_result.rb
 - spec/spec_helper.rb