mysql_manager 1.0.3 → 1.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7db617b7015f5a675990bef408681d12e9333c1e
+  data.tar.gz: 6c130b35e26811fd5defcdfba76efb38147aef0f
+SHA512:
+  metadata.gz: ee25ebbb9fc2c754fb96b5edbe9ef157e6483be8954ab928ba008a77f1491686e1e523d9fb1afa2dbae7bb18df5cb6855115e2487f782f6dc068151db7a5eaa3
+  data.tar.gz: cb8de0cc6774c0762e963ccbd92fa2709e8d4c13c8a4528455992299e4c11ea853aad089eb4f4a74f677ef4b32ed99bcc5612392a7ea0ea4d30a343440832ae2

data/README.md

@@ -22,7 +22,7 @@ Or install it yourself as:
 
 ## Usage
 
-    Usage: mysql-manager options
+    Usage: bin/mysql-manager options
             --kill                       Kill queries based on specified criteria
             --kill:max-query-time TIME   Kill queries that have been running for more than TIME
             --kill:user USER             Kill queries matching USER (repeatable)
@@ -48,10 +48,17 @@ Or install it yourself as:
             --log:file FILE              Write logs to FILE (default: STDERR)
             --log:age DAYS               Rotate logs after DAYS pass (default: 7)
             --log:size SIZE              Rotate logs after the grow past SIZE bytes
+            --hotcopy                    Perform a hotcopy
+            --hotcopy:data-dir PATH      Rsync mysql data dir from PATH (default: /var/lib/mysql)
+            --hotcopy:backup-dir PATH    Rsync to PATH (default: /tmp/mysql)
+            --hotcopy:rsync-args ARGS    Arguments to pass to rsync (default: -av)
+            --hotcopy:rsync-bin bin      Rsync executable path (default: rsync)
+            --hotcopy:rsync-ttl ttl      Acceptable rsync execution time before performing table locks (default: 60)
             --dry-run                    Do not run statements which affect the state of the database when executed
         -V, --version                    Display version information
         -h, --help                       Display this screen
 
+
 ## How matching is done
 
 If you specify a simple string as a parameter (e.g. `--kill:user api`), this is interpreted as an exactly matching
@@ -78,6 +85,16 @@ Reload `/etc/my.cnf` without restarting MySQL:
 
     mysql-manager --reload-my-cnf --reload-my-cnf:config /etc/my.cnf --log:level DEBUG
 
+Perform a hotcopy backup of the running mysql database using rsync to a remote server. It will repeated rsync the folder without locks until syncs take less than 30 seconds.
+
+    mysql-manager --hotcopy \
+                  --hotcopy:data-dir /var/lib/mysql/ \
+                  --hotcopy:backup-dir user@remote.host.com:/tmp/mysql/ \
+                  --hotcopy:rsync-args "-av --exclude=*.err" \
+                  --hotcopy:rsync-ttl 30 \
+                  --db:user root \
+                  --db:pass $MYSQL_ROOT_PASSWORD
+
 ## Contributors
 
 https://github.com/osterman/mysql_manager/graphs/contributors

data/lib/mysql_manager/command_line.rb

@@ -40,6 +40,7 @@ module MysqlManager
       @options[:kill] = {}
       @options[:reload_my_cnf] = {}
       @options[:skip_replication_errors] = {}
+      @options[:hotcopy] = {}
 
       begin
         @optparse = OptionParser.new do |opts|
@@ -178,6 +179,40 @@ module MysqlManager
             @options[:log][:size] = size.to_i
           end
 
+          #
+          # Hotcopy
+          #
+
+          @options[:hotcopy][:execute] = false
+          opts.on( '--hotcopy', 'Perform a hotcopy') do
+            @options[:hotcopy][:execute] = true
+          end
+
+          @options[:hotcopy][:data_dir] = '/var/lib/mysql'
+          opts.on( '--hotcopy:data-dir PATH', "Rsync mysql data dir from PATH (default: #{@options[:hotcopy][:data_dir]})") do |path|
+            @options[:hotcopy][:data_dir] = path.to_s
+          end
+
+          @options[:hotcopy][:backup_dir] = '/tmp/mysql'
+          opts.on( '--hotcopy:backup-dir PATH', "Rsync to PATH (default: #{@options[:hotcopy][:backup_dir]})") do |path|
+            @options[:hotcopy][:backup_dir] = path.to_s
+          end
+
+          @options[:hotcopy][:rsync_args] = '-av'
+          opts.on( '--hotcopy:rsync-args ARGS', "Arguments to pass to rsync (default: #{@options[:hotcopy][:rsync_args]})") do |args|
+            @options[:hotcopy][:rsync_args] = Shellwords.split(args.to_s)
+          end
+
+          @options[:hotcopy][:rsync_bin] = 'rsync'
+          opts.on( '--hotcopy:rsync-bin bin', "Rsync executable path (default: #{@options[:hotcopy][:rsync_bin]})") do |bin|
+            @options[:hotcopy][:rsync_bin] = bin.to_s
+          end
+
+          @options[:hotcopy][:rsync_ttl] = 60
+          opts.on( '--hotcopy:rsync-ttl ttl', "Acceptable rsync execution time before performing table locks (default: #{@options[:hotcopy][:rsync_ttl]})") do |ttl|
+            @options[:hotcopy][:rsync_ttl] = ttl.to_i
+          end
+
           # 
           # General options
           #
@@ -234,6 +269,9 @@ module MysqlManager
           when :reload_my_cnf
             @log.debug("about to call reload_my_cnf")
             @utilities.reload_my_cnf(options)
+          when :hotcopy
+            @log.debug("about to call hotcopy")
+            @utilities.hotcopy(options)
           end
         rescue DBI::DatabaseError => e
           @log.fatal(e.message)

data/lib/mysql_manager/utilities.rb

@@ -20,6 +20,8 @@
 require 'dbi'
 require 'logger'
 require 'parseconfig'
+require 'shellwords'
+require 'open3'
 
 module MysqlManager
   class FileNotFoundException < Exception; end
@@ -214,5 +216,72 @@ module MysqlManager
         @log.error("Error code: #{e.err}: #{e.errstr}")
       end
     end
+
+    def run(cmd)
+      stdout, stderr, status = Open3.capture3(Shellwords.join(cmd))
+      if status != 0
+        puts stderr
+        raise Exception.new("execution of command failed")
+      end
+      return status
+    end
+
+    def hotcopy(options = {})
+      begin
+        t_start = Time.now.to_f
+        # ensure mysql directory exists
+        unless File.directory?(options[:data_dir])
+          raise FileNotFoundException.new("Unable to open mysql data dir #{options[:data_dir]}")
+        end
+
+        rsync = []
+        rsync << options[:rsync_bin]
+        rsync << options[:rsync_args]
+        rsync << options[:data_dir] + '/'
+        rsync << options[:backup_dir] + '/'
+
+        # create target directory if it does not already exist
+        unless File.directory?(options[:backup_dir])
+          @log.info("Creating #{options[:backup_dir]}")
+          run ['mkdir', '-p', options[:backup_dir]]
+        end
+
+        rsync.flatten!
+
+        t_rsync = Time.now.to_f
+        t_rsync_elapsed = Time.now.to_f
+        while t_rsync_elapsed > options[:rsync_ttl]
+          # first do a raw rsync without table locks
+          @log.info("Performing first-pass rsync without table locks")
+          run rsync
+          t_rsync_elapsed = Time.now.to_f - t_rsync
+          @log.info("Rsync took #{t_rsync_elapsed} seconds")
+          sleep 1
+          t_rsync = Time.now.to_f
+        end
+
+        # then flush tables with read lock
+        @log.info("Locking tables")
+        t_lock = Time.now.to_f
+        @dbh.do("FLUSH TABLES WITH READ LOCK")  unless @dry_run
+
+        # re-sync the delats
+        @log.info("Performing second-pass rsync with table locks")
+        run rsync
+      
+        # release locks
+        @log.info("Unlocking tables")
+        @dbh.do('UNLOCK TABLES')
+        @log.info("Total lock time #{Time.now.to_f - t_lock} seconds")
+        @log.info("Total elapsed time #{Time.now.to_f - t_start} seconds")
+
+      rescue DBI::DatabaseError => e
+        @log.debug(@dbh.last_statement)
+        @log.warn(e.message)
+      end
+    end
+
+ 
+
   end
 end

data/lib/mysql_manager/version.rb

@@ -18,5 +18,5 @@
 # along with MySQL Manager.  If not, see <http://www.gnu.org/licenses/>.
 #
 module MysqlManager
-  VERSION = "1.0.3"
+  VERSION = "1.0.4"
 end

data/mysql_manager.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = MysqlManager::VERSION
   gem.add_runtime_dependency "parseconfig", ">= 1.0.2"
+  gem.add_runtime_dependency "mysql2", ">= 0.3.18"
   gem.add_runtime_dependency "dbi", ">= 0.4.5"
   gem.add_runtime_dependency "dbd-mysql", ">= 0.4.4"
 end

metadata

@@ -1,77 +1,79 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mysql_manager
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 0
-  - 3
-  version: 1.0.3
+version: !ruby/object:Gem::Version
+  version: 1.0.4
 platform: ruby
-authors: 
+authors:
 - Erik Osterman
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-08-09 00:00:00 -07:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2015-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: parseconfig
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
   prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 0
-        - 2
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
         version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.18
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: dbi
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 4
-        - 5
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.18
+- !ruby/object:Gem::Dependency
+  name: dbi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
         version: 0.4.5
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: dbd-mysql
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 4
-        - 4
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.5
+- !ruby/object:Gem::Dependency
+  name: dbd-mysql
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
         version: 0.4.4
   type: :runtime
-  version_requirements: *id003
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.4
 description: MySQL Manager is a utility to perform routine tasks on a MySQL database.
-email: 
+email:
 - e@osterman.com
-executables: 
+executables:
 - mysql-manager
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - Gemfile
 - LICENSE
@@ -83,37 +85,30 @@ files:
 - lib/mysql_manager/utilities.rb
 - lib/mysql_manager/version.rb
 - mysql_manager.gemspec
-has_rdoc: true
 homepage: https://github.com/osterman/mysql_manager
-licenses: 
+licenses:
 - GPL3
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 2.0.14
 signing_key: 
-specification_version: 3
-summary: MySQL Manager is a utility to perform routine tasks such as restoring replication after errors, killing queries based on a set of criteria, or reloading my.cnf without restarting the database.
+specification_version: 4
+summary: MySQL Manager is a utility to perform routine tasks such as restoring replication
+  after errors, killing queries based on a set of criteria, or reloading my.cnf without
+  restarting the database.
 test_files: []
-