mysql-slaver 0.1.0

data/Gemfile

@@ -0,0 +1,13 @@
+source "https://rubygems.org"
+
+gem "thor"
+
+group :development, :test do
+  gem "rdoc"
+  gem "rdoc-data"
+  gem "rake"
+  gem "gem-this"
+  gem "gemcutter"
+  gem "rspec"
+  gem "ruby-debug"
+end

data/Gemfile.lock

@@ -0,0 +1,43 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    columnize (0.3.6)
+    diff-lcs (1.2.5)
+    gem-this (0.3.7)
+    gemcutter (0.7.1)
+    json (1.8.1)
+    linecache (0.46)
+      rbx-require-relative (> 0.0.4)
+    rake (10.3.0)
+    rbx-require-relative (0.0.9)
+    rdoc (4.1.1)
+      json (~> 1.4)
+    rdoc-data (4.0.1)
+      rdoc (~> 4.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.8)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.6)
+    ruby-debug (0.10.4)
+      columnize (>= 0.1)
+      ruby-debug-base (~> 0.10.4.0)
+    ruby-debug-base (0.10.4)
+      linecache (>= 0.3)
+    thor (0.19.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  gem-this
+  gemcutter
+  rake
+  rdoc
+  rdoc-data
+  rspec
+  ruby-debug
+  thor

data/README.md

@@ -0,0 +1,43 @@
+Tool to setup a mysql replication slave by copying data and master status from a remote mysql master over ssh.
+
+USAGE
+
+    mysql_slaver help enslave
+
+    mysql_slaver enslave --database=DATABASE --master-host=MASTER_HOST --replication-password=REPLICATION_PASSWORD --replication-user=REPLICATION_USER
+
+BLOGPOST
+
+  http://digitalronin.github.io/2014/04/16/mysql-slaver-gem-setup-mysql-replication/
+
+ASSUMPTIONS/PRE-REQUISITES
+
+* localhost is configured as a mysql replication slave
+* the current localhost user can ssh to the db master
+* your mysql administrator user is called 'root', locally and on the db master
+* root user has the same password on this host and the master server
+* mysql is on the local user''s path
+* mysql and mysqldump are on the remote ssh user''s path
+* replication permissions from the local host to the db master are already setup
+* mysql is running on the default port (3306)
+* ssh is on the current user''s path
+* db character set is UTF-8
+* any ssh config settings for the host are set in a ~/.ssh/config file
+
+CAVEATS
+
+* destructively replaces the target database on localhost with no backup
+
+TODO
+
+* accept (and insist on) command-line parameters
+* add a "no copying" mode that only updates master log file and position
+* add a "dry-run" mode
+* package as a gem
+* check ssh connection and permissions
+* check replication permissions
+* check slave is setup as a replication slave (i.e. it has a mysql server id)
+* allow overriding the mysql port
+* allow overriding the mysql root user
+* allow different root user passwords on slave and master
+* allow ssh options

data/Rakefile

@@ -0,0 +1,80 @@
+require "rubygems"
+require "rubygems/package_task"
+require "rdoc/task"
+
+require "rspec"
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = %w(--format documentation --colour)
+end
+
+
+task :default => ["spec"]
+
+# This builds the actual gem. For details of what all these options
+# mean, and other ones you can add, check the documentation here:
+#
+#   http://rubygems.org/read/chapter/20
+#
+spec = Gem::Specification.new do |s|
+
+  # Change these as appropriate
+  s.name              = "mysql-slaver"
+  s.version           = "0.1.0"
+  s.summary           = "What this thing does"
+  s.author            = "David Salgado"
+  s.email             = "david@digitalronin.com"
+  s.homepage          = "http://yoursite.example.com"
+
+  s.has_rdoc          = true
+  s.extra_rdoc_files  = %w(README.md)
+  s.rdoc_options      = %w(--main README.md)
+
+  # Add any extra files to include in the gem
+  s.files             = %w(Gemfile Gemfile.lock Rakefile README.md) + Dir.glob("{bin,spec,lib}/**/*")
+  s.executables       = FileList["bin/**"].map { |f| File.basename(f) }
+  s.require_paths     = ["lib"]
+
+  # If you want to depend on other gems, add them here, along with any
+  # relevant versions
+  # s.add_dependency("some_other_gem", "~> 0.1.0")
+
+  # If your tests use any gems, include them here
+  s.add_development_dependency("rspec")
+end
+
+# This task actually builds the gem. We also regenerate a static
+# .gemspec file, which is useful if something (i.e. GitHub) will
+# be automatically building a gem for this project. If you're not
+# using GitHub, edit as appropriate.
+#
+# To publish your gem online, install the 'gemcutter' gem; Read more 
+# about that here: http://gemcutter.org/pages/gem_docs
+Gem::PackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "Build the gemspec file #{spec.name}.gemspec"
+task :gemspec do
+  file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
+  File.open(file, "w") {|f| f << spec.to_ruby }
+end
+
+# If you don't want to generate the .gemspec file, just remove this line. Reasons
+# why you might want to generate a gemspec:
+#  - using bundler with a git source
+#  - building the gem without rake (i.e. gem build blah.gemspec)
+#  - maybe others?
+task :package => :gemspec
+
+# Generate documentation
+RDoc::Task.new do |rd|
+  rd.main = "README.md"
+  rd.rdoc_files.include("README.md", "lib/**/*.rb")
+  rd.rdoc_dir = "rdoc"
+end
+
+desc 'Clear out RDoc and generated packages'
+task :clean => [:clobber_rdoc, :clobber_package] do
+  rm "#{spec.name}.gemspec"
+end

data/bin/mysql_slaver

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'thor'
+require 'lib/mysql_slaver'
+
+module MysqlSlaver
+  class CLI < Thor
+    option :master_host,           :required => true, :desc => "The server which will be the replication master, for this slave"
+    option :database,              :required => true, :desc => "The database to copy from the master"
+    option :replication_user,      :required => true, :desc => "DB user (on the master host), with replication permissions"
+    option :replication_password,  :required => true, :desc => "DB password for the replication user"
+    option :root_password,         :desc     => "Password for the mysql root user (on both master and slave)"
+    desc "enslave", "start mysql replication to this host from a master"
+    long_desc <<-LONGDESC
+    LONGDESC
+    def enslave
+      MysqlSlaver::Slaver.new(
+          :master_host          => options[:master_host],
+          :mysql_root_password  => options[:root_password],
+          :database             => options[:database],
+          :replication_user     => options[:replication_user],
+          :replication_password => options[:replication_password]
+      ).enslave!
+    end
+  end
+end
+
+MysqlSlaver::CLI.start(ARGV)

data/lib/mysql_slaver/db_copier.rb

@@ -0,0 +1,23 @@
+module MysqlSlaver
+  class DbCopier
+    include MysqlCommand
+
+    attr_accessor :master_host, :mysql_root_password, :database, :executor
+
+    def initialize(params)
+      @master_host         = params.fetch(:master_host)
+      @mysql_root_password = params.fetch(:mysql_root_password, '')
+      @database            = params.fetch(:database)
+      @executor            = params.fetch(:executor) { Executor.new }
+    end
+
+    def copy!
+      executor.execute mysql_command("stop slave", mysql_root_password)
+      cmd = mysqldump(master_host, database, mysql_root_password)
+      dump_cmd = executor.ssh_command(cmd, master_host)
+      load_cmd = ['mysql', mysql_credentials('root', mysql_root_password), database].join(' ')
+      command = [dump_cmd, load_cmd].join(' | ')
+      executor.execute command
+    end
+  end
+end

data/lib/mysql_slaver/executor.rb

@@ -0,0 +1,15 @@
+module MysqlSlaver
+  class Executor
+    include Logger
+
+    def ssh_command(cmd, host)
+      "ssh #{host} '#{cmd}'"
+    end
+
+    def execute(cmd)
+      string = cmd.is_a?(Array) ? cmd.join('; ') : cmd
+      log "CMD: #{string}"
+      `#{string}`
+    end
+  end
+end

data/lib/mysql_slaver/logger.rb

@@ -0,0 +1,7 @@
+module MysqlSlaver
+  module Logger
+    def log(msg)
+      puts [Time.now.strftime("%Y-%m-%d %H:%M:%S"), msg].join(' ')
+    end
+  end
+end

data/lib/mysql_slaver/master_changer.rb

@@ -0,0 +1,31 @@
+module MysqlSlaver
+  class MasterChanger
+    include MysqlCommand
+
+    attr_accessor :master_host, :mysql_root_password, :replication_user, :replication_password, :executor
+
+    def initialize(params)
+      @master_host          = params.fetch(:master_host)
+      @mysql_root_password  = params.fetch(:mysql_root_password, '')
+      @replication_user     = params.fetch(:replication_user)
+      @replication_password = params.fetch(:replication_password)
+      @executor             = params.fetch(:executor) { Executor.new }
+    end
+
+    def change!(status)
+      cmds = [
+        'stop slave',
+        change_master(status),
+        'start slave'
+      ]
+      cmd = mysql_command(cmds.join('; '), mysql_root_password)
+      executor.execute cmd
+    end
+
+    private
+
+    def change_master(status)
+      %[CHANGE MASTER TO MASTER_LOG_FILE='#{status[:file]}', MASTER_LOG_POS=#{status[:position]}, MASTER_HOST='#{master_host}', MASTER_USER='#{replication_user}', MASTER_PASSWORD='#{replication_password}']
+    end
+  end
+end

data/lib/mysql_slaver/mysql_command.rb

@@ -0,0 +1,19 @@
+module MysqlSlaver
+  module MysqlCommand
+    def mysql_credentials(user, password)
+      rtn = "-u #{user} "
+      rtn << "-p #{password} " unless password.to_s == ""
+      rtn
+    end
+
+    def mysql_command(cmd, password)
+      creds = mysql_credentials('root', password)
+      %[mysql #{creds} -e "#{cmd}"]
+    end
+
+    def mysqldump(host, database, password)
+      creds = mysql_credentials('root', password)
+      %[mysqldump -h #{host} #{creds} --master-data --single-transaction --quick --skip-add-locks --skip-lock-tables --default-character-set=utf8 --compress #{database}]
+    end
+  end
+end

data/lib/mysql_slaver/slaver.rb

@@ -0,0 +1,45 @@
+# Setup the current server as a replication slave of a master mysql server,
+# by dumping and loading a database, and issuing the appropriate change master
+# command to mysql.
+# This assumes that the current user can issue ssh commands to the master
+# server (to get its master status, and to dump and load data over the ssh
+# connection)
+module MysqlSlaver
+  class Slaver
+    attr_reader :status_fetcher, :data_copier, :master_changer
+
+    def initialize(params)
+      mysql_root_password  = params.fetch(:mysql_root_password, '')
+
+      @status_fetcher = params.fetch(:status_fetcher) {
+        StatusFetcher.new(
+          :master_host         => params.fetch(:master_host),
+          :mysql_root_password => mysql_root_password
+        )
+      }
+
+      @data_copier = params.fetch(:data_copier) {
+        DbCopier.new(
+          :master_host         => params.fetch(:master_host),
+          :mysql_root_password => mysql_root_password,
+          :database            => params.fetch(:database)
+        )
+      }
+
+      @master_changer = params.fetch(:master_changer) {
+        MasterChanger.new(
+          :master_host          => params.fetch(:master_host),
+          :mysql_root_password  => mysql_root_password,
+          :replication_user     => params.fetch(:replication_user),
+          :replication_password => params.fetch(:replication_password)
+        )
+      }
+    end
+
+    def enslave!
+      master_status = status_fetcher.status
+      data_copier.copy!
+      master_changer.change!(master_status)
+    end
+  end
+end

data/lib/mysql_slaver/status_fetcher.rb

@@ -0,0 +1,39 @@
+module MysqlSlaver
+  class StatusFetcher
+    include Logger
+    include MysqlCommand
+
+    attr_accessor :master_host, :mysql_root_password, :executor
+
+    def initialize(params)
+      @master_host          = params.fetch(:master_host)
+      @mysql_root_password  = params.fetch(:mysql_root_password, '')
+      @executor             = params.fetch(:executor) { Executor.new }
+    end
+
+    def status
+      cmd = mysql_command("show master status\\G", mysql_root_password)
+      data = executor.execute executor.ssh_command(cmd, master_host)
+      rtn = parse data
+      log "MASTER STATUS - file: #{rtn[:file]}, position: #{rtn[:position]}"
+      rtn
+    end
+
+    private
+
+    def parse(text)
+      file = nil
+      position = nil
+      text.split("\n").each do |line|
+        case line
+        when /File: (mysql-bin\.\d+)/
+          file = $1
+        when /Position: (\d+)/
+          position = $1
+        end
+      end
+      {:file => file, :position => position}
+    end
+
+  end
+end

data/lib/mysql_slaver.rb

@@ -0,0 +1,7 @@
+require 'lib/mysql_slaver/logger'
+require 'lib/mysql_slaver/executor'
+require 'lib/mysql_slaver/mysql_command'
+require 'lib/mysql_slaver/slaver'
+require 'lib/mysql_slaver/status_fetcher'
+require 'lib/mysql_slaver/db_copier'
+require 'lib/mysql_slaver/master_changer'

data/spec/mysql_slaver/db_copier_spec.rb

@@ -0,0 +1,39 @@
+require 'spec/spec_helper'
+
+module MysqlSlaver
+  describe DbCopier do
+    let(:executor) { double(Executor, :execute => true, :ssh_command => "dummy-ssh-command") }
+
+    let(:params) {
+      {
+        :master_host         => 'my.db.host',
+        :mysql_root_password => 'supersekrit',
+        :database            => 'myappdb',
+        :executor            => executor
+      }
+    }
+    subject(:copier) { described_class.new(params) }
+
+    describe "#copy!" do
+      it "stops slave" do
+        copier.copy!
+        stop = %[mysql -u root -p supersekrit  -e "stop slave"]
+        expect(executor).to have_received(:execute).with(stop)
+      end
+
+      it "loads data" do
+        dump_and_load = "dummy-ssh-command | mysql -u root -p supersekrit  myappdb"
+        expect(executor).to receive(:execute).once.ordered.with(dump_and_load)
+        copier.copy!
+      end
+
+      context "dumping" do
+        it "issues mysqldump over ssh" do
+          dump = "mysqldump -h my.db.host -u root -p supersekrit  --master-data --single-transaction --quick --skip-add-locks --skip-lock-tables --default-character-set=utf8 --compress myappdb"
+          expect(executor).to receive(:ssh_command).with(dump, 'my.db.host')
+          copier.copy!
+        end
+      end
+    end
+  end
+end

data/spec/mysql_slaver/executor_spec.rb

@@ -0,0 +1,13 @@
+require 'spec/spec_helper'
+
+module MysqlSlaver
+  describe Executor do
+    subject(:executor) { described_class.new }
+
+    describe "#ssh_command" do
+      it "formats command" do
+        expect(executor.ssh_command("foo", "myhost")).to eq("ssh myhost 'foo'")
+      end
+    end
+  end
+end

data/spec/mysql_slaver/master_changer_spec.rb

@@ -0,0 +1,27 @@
+require 'spec/spec_helper'
+
+module MysqlSlaver
+  describe MasterChanger do
+    let(:executor) { double(Executor, :execute => true) }
+
+    let(:status) { {:file => 'mysql-bin.001555', :position => 18426246} }
+    let(:params) {
+      {
+        :master_host          => 'my.db.host',
+        :mysql_root_password  => 'supersekrit',
+        :replication_user     => 'repluser',
+        :replication_password => 'replpassword',
+        :executor             => executor
+      }
+    }
+    subject(:changer) { described_class.new(params) }
+
+    describe "#change!" do
+      it "executes multi-part mysql command" do
+        change_cmd = %[mysql -u root -p supersekrit  -e "stop slave; CHANGE MASTER TO MASTER_LOG_FILE='mysql-bin.001555', MASTER_LOG_POS=18426246, MASTER_HOST='my.db.host', MASTER_USER='repluser', MASTER_PASSWORD='replpassword'; start slave"]
+        changer.change!(status)
+        expect(executor).to have_received(:execute).with(change_cmd)
+      end
+    end
+  end
+end

data/spec/mysql_slaver/slaver_spec.rb

@@ -0,0 +1,70 @@
+require 'spec/spec_helper'
+
+module MysqlSlaver
+  describe Slaver do
+    subject(:slaver) { described_class.new(params) }
+
+    describe "#enslave!" do
+      let(:status_fetcher) { double(StatusFetcher, :status  => ())   }
+      let(:data_copier)    { double(DbCopier, :copy!        => true) }
+      let(:master_changer) { double(MasterChanger, :change! => true) }
+
+      let(:params) {
+        {
+          :status_fetcher => status_fetcher,
+          :data_copier    => data_copier,
+          :master_changer => master_changer
+        }
+      }
+
+      it "fetches master status" do
+        slaver.enslave!
+        expect(status_fetcher).to have_received(:status)
+      end
+
+      it "copies data" do
+        slaver.enslave!
+        expect(data_copier).to have_received(:copy!)
+      end
+
+      it "changes master status" do
+        slaver.enslave!
+        expect(master_changer).to have_received(:change!)
+      end
+
+    end
+
+    context "instantiating collaborators" do
+      let(:params) {
+        {
+          :master_host          => 'my.db.host',
+          :mysql_root_password  => 'supersekrit',
+          :database             => 'myappdb',
+          :replication_user     => 'repluser',
+          :replication_password => 'replpassword'
+        }
+      }
+
+      it "instantiates a status fetcher" do
+        fetcher = slaver.status_fetcher
+        expect(fetcher.master_host).to eq('my.db.host')
+        expect(fetcher.mysql_root_password).to eq('supersekrit')
+      end
+
+      it "instantiates a data copier" do
+        copier = slaver.data_copier
+        expect(copier.master_host).to eq('my.db.host')
+        expect(copier.mysql_root_password).to eq('supersekrit')
+        expect(copier.database).to eq('myappdb')
+      end
+
+      it "instantiates a master changer" do
+        changer = slaver.master_changer
+        expect(changer.master_host).to eq('my.db.host')
+        expect(changer.mysql_root_password).to eq('supersekrit')
+        expect(changer.replication_user).to eq('repluser')
+        expect(changer.replication_password).to eq('replpassword')
+      end
+    end
+  end
+end

data/spec/mysql_slaver/status_fetcher_spec.rb

@@ -0,0 +1,45 @@
+require 'spec/spec_helper'
+
+module MysqlSlaver
+  describe StatusFetcher do
+    let(:executor) { double(Executor, :execute => "", :ssh_command => "dummy-ssh-command") }
+
+    let(:params) {
+      {
+        :master_host          => 'my.db.host',
+        :mysql_root_password  => 'supersekrit',
+        :executor             => executor
+      }
+    }
+    subject(:fetcher) { described_class.new(params) }
+
+    before do
+      fetcher.stub(:log)
+    end
+
+    describe "#status" do
+      let(:output) { <<EOF
+*************************** 1. row ***************************
+            File: mysql-bin.003219
+        Position: 37065270
+    Binlog_Do_DB:
+Binlog_Ignore_DB:
+EOF
+      }
+
+      before do
+        executor.stub(:execute => output)
+      end
+        
+      it "executes show master command over ssh" do
+        show_master = %[mysql -u root -p supersekrit  -e "show master status\\G"]
+        fetcher.status
+        expect(executor).to have_received(:ssh_command).with(show_master, 'my.db.host')
+      end
+
+      it "parses show master output" do
+        expect(fetcher.status).to eq({:file => 'mysql-bin.003219', :position => '37065270'})
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'ruby-debug'
+Debugger.start
+
+require 'lib/mysql_slaver'

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification 
+name: mysql-slaver
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- David Salgado
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2014-04-16 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: 
+email: david@digitalronin.com
+executables: 
+- mysql_slaver
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- README.md
+- bin/mysql_slaver
+- spec/mysql_slaver/db_copier_spec.rb
+- spec/mysql_slaver/executor_spec.rb
+- spec/mysql_slaver/master_changer_spec.rb
+- spec/mysql_slaver/slaver_spec.rb
+- spec/mysql_slaver/status_fetcher_spec.rb
+- spec/spec_helper.rb
+- lib/mysql_slaver/db_copier.rb
+- lib/mysql_slaver/executor.rb
+- lib/mysql_slaver/logger.rb
+- lib/mysql_slaver/master_changer.rb
+- lib/mysql_slaver/mysql_command.rb
+- lib/mysql_slaver/slaver.rb
+- lib/mysql_slaver/status_fetcher.rb
+- lib/mysql_slaver.rb
+homepage: https://github.com/digitalronin/mysql-slaver
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.md
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Setup mysql replication on a slave (localhost), from a remote master, over SSH
+test_files: []
+