mys3ql 1.0.2 → 1.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 86d0a66ce7dab918b298c22eb4d667df2cb9ac7c
+  data.tar.gz: cd9527db20ae5a07349b3456e544a0aa3552f9bd
+SHA512:
+  metadata.gz: 25abe9f3cc7daee801c8402dbd07b8827608d139ed4a2ca95ec089ed1a7d4335dfbfaa8fd30f10ea06118159d3513fa90c866ac9fa09976b48dc8d05645ac105
+  data.tar.gz: 340d29d1230eda51d4408b8f5fb3597ed7da31276171608469092c10fbe04ceb5f3e0640f3758cbdbe14c2af99804bc5ebb81dd27e6097c661c492d4339d56f4

data/README.md

@@ -2,6 +2,8 @@
 
 Simple backup of your MySQL database onto Amazon S3.
 
+See [Example: mysqldump + mysqlbinlog for Backup and Restore](https://dev.mysql.com/doc/refman/5.7/en/mysqlbinlog-backup.html#mysqlbinlog-backup-example).
+
 
 ## Quick start
 
@@ -19,6 +21,12 @@ To restore from the latest backup (plus binlogs if present):
 
     $ mys3ql restore
 
+To restore a recent subset of binlogs:
+
+    $ mys3ql restore --after NUMBER
+
+– where NUMBER is a 6-digit binlog file number.
+
 By default mys3ql looks for a configuration file at `~/.mys3ql`.  You can override this like so:
 
     $ mys3ql [command] -c FILE
@@ -79,20 +87,8 @@ N.B. the binary logs contain updates to all the databases on the server.  This m
 Marc-André Cournoyer's [mysql_s3_backup](https://github.com/macournoyer/mysql_s3_backup).
 
 
-## To Do
-
-- tests ;)
-- remove old dump files (s3)
-- (restore from non-latest dump)
-
-
-## Questions, Problems, Feedback
-
-Please use the GitHub [issue tracker](https://github.com/airblade/mys3ql/issues) or email me.
-
-
 ## Intellectual property
 
-Copyright 2011 Andy Stewart (boss@airbladesoftware.com).
+Copyright 2011-2021 Andy Stewart (boss@airbladesoftware.com).
 
 Released under the MIT licence.

data/bin/mys3ql

@@ -1,35 +1,33 @@
-#!/usr/bin/env ruby-local-exec
-
-lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
-$LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
+#!/usr/bin/env ruby
 
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'mys3ql'
-require 'main'
+require 'optparse'
 
-Main do
-  # consider using modes as/when we need command-specific arguments (e.g. restore specific backup)
-  argument 'command' do
-    validate { |command| %w[ full incremental restore ].include? command }
-    description 'specifies the operation to perform [full | incremental | restore]'
-  end
+params = {}
+op = OptionParser.new do |opts|
+  opts.banner = 'Usage: mys3ql <full|incremental|restore> [<options>]'
 
-  option 'config', 'c' do
-    argument :required
-    description 'load configuration from YAML file'
-    defaults '~/.mys3ql'
-  end
+  opts.separator ''
+  opts.separator 'Common options:'
 
-  option 'debug', 'd' do
-    description 'be verbose'
+  opts.on('-c', '--config CONFIG', 'Load configuration from YAML file (default ~/.mys3ql)') { |v| params[:config] = v }
+  opts.on('-d', '--debug', 'Be verbose') { |v| params[:debug] = v }
+  opts.on '-v', '--version', 'Print version' do
+    puts "mys3ql v#{Mys3ql::VERSION}"
+    exit
   end
 
-  option 'version', 'v'
+  opts.separator ''
+  opts.separator 'restore options:'
+  opts.on('-a', '--after NUMBER', 'Use only the subset of binary logs after NUMBER') { |v| params[:after] = v }
+end
+op.parse! ARGV
 
-  def run
-    if params[:version].given?
-      puts "mys3ql v#{Mys3ql::VERSION}"
-      exit
-    end
-    Mys3ql::Conductor.run params[:command].value, params[:config].value, params[:debug].given?
-  end
+params[:command] = ARGV[0]
+unless %w[full incremental restore].include? params[:command]
+  puts op.help
+  exit 1
 end
+
+Mys3ql::Conductor.run params

data/lib/mys3ql/conductor.rb

@@ -6,10 +6,16 @@ require 'mys3ql/s3'
 module Mys3ql
   class Conductor
 
-    def self.run(command, config, debug)
-      conductor = Conductor.new(config)
-      conductor.debug = debug
-      conductor.send command
+    def self.run(args)
+      conductor = Conductor.new args.fetch(:config, nil)
+      conductor.debug = args.fetch(:debug, false)
+
+      command = args.fetch(:command)
+      if command == 'restore'
+        conductor.restore args.fetch(:after, nil)
+      else
+        conductor.send command
+      end
     end
 
     def initialize(config_file = nil)
@@ -18,6 +24,10 @@ module Mys3ql
       @s3 = S3.new @config
     end
 
+    # Dumps the database and uploads it to S3.
+    # Copies the uploaded file to the key :latest.
+    # Deletes binary logs from the file system.
+    # Deletes binary logs from S3.
     def full
       @mysql.dump
       @s3.store @mysql.dump_file
@@ -25,30 +35,49 @@ module Mys3ql
       @s3.delete_bin_logs
     end
 
+    # Uploads mysql's binary logs to S3.
+    # The binary logs are left on the file system.
+    # Log files already on S3 are not re-uploaded.
     def incremental
       @mysql.each_bin_log do |log|
         @s3.store log, false
       end
     end
 
-    # for now only restore from latest
-    def restore
-      # get latest dump
-      with_temp_file do |file|
-        @s3.retrieve :latest, file
-        @mysql.restore file
+    # When 'after' is nil:
+    #
+    # - downloads the latest dump from S3 and loads it into the database;
+    # - downloads each binary log from S3 and loads it into the database.
+    #
+    # When 'after' is given:
+    #
+    # - downloads each binary log following 'after' from S3 and loads it into the database.
+    #
+    # Downloaded files are removed from the file system.
+    def restore(after = nil)
+      unless after
+        # get latest dump
+        with_temp_file do |file|
+          @s3.retrieve :latest, file
+          @mysql.restore file
+        end
       end
 
-      # apply subsequent bin logs
-      @s3.each_bin_log do |log|
-        with_temp_file do |file|
-          @s3.retrieve log, file
-          @mysql.apply_bin_log file
+      # apply bin logs
+      begin
+        tmpfiles = []
+        @s3.each_bin_log(after) do |log|
+          file = Tempfile.new 'mys3ql'
+          tmpfiles << file
+          @s3.retrieve log, file.path
         end
+        @mysql.apply_bin_logs tmpfiles.map(&:path)
+      ensure
+        tmpfiles.each &:close!
       end
 
       # NOTE: not sure about this:
-      puts "You might want to flush mysql's logs..."
+      # puts "You might want to flush mysql's logs..."
     end
 
     def debug=(val)

data/lib/mys3ql/mysql.rb

@@ -13,8 +13,10 @@ module Mys3ql
     #
 
     def dump
+      # --master-data=2         include the current binary log coordinates in the log file
+      # --delete-master-logs    delete binary log files
       cmd  = "#{@config.bin_path}mysqldump"
-      cmd += ' --quick --single-transaction --create-options'
+      cmd += ' --quick --single-transaction --create-options --no-tablespaces'
       cmd += ' --flush-logs --master-data=2 --delete-master-logs' if binary_logging?
       cmd += cli_options
       cmd += " | gzip > #{dump_file}"
@@ -36,14 +38,18 @@ module Mys3ql
 
     # flushes logs, yields each bar the last to the block
     def each_bin_log(&block)
+      # FLUSH LOGS    Closes and reopens any log file, including binary logs,
+      #               to which the server is writing.  For binary logs, the sequence
+      #               number of the binary log file is incremented by one relative to
+      #               the previous file.
+      #               https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-logs
+      #               https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-binary-logs
       execute 'flush logs'
       logs = Dir.glob("#{@config.bin_log}.[0-9]*").sort_by { |f| f[/\d+/].to_i }
-      logs_to_backup = logs[0..-2]  # all logs except the last, which is in use
+      logs_to_backup = logs[0..-2]  # all logs except the last, which is newly created
       logs_to_backup.each do |log_file|
         yield log_file
       end
-      # delete binlogs from file system
-      #execute "purge master logs to '#{File.basename(logs[-1])}'"
     end
 
     #
@@ -54,8 +60,8 @@ module Mys3ql
       run "gunzip -c #{file} | #{@config.bin_path}mysql #{cli_options}"
     end
 
-    def apply_bin_log(file)
-      cmd  = "#{@config.bin_path}mysqlbinlog --database=#{@config.database} #{file}"
+    def apply_bin_logs(*files)
+      cmd  = "#{@config.bin_path}mysqlbinlog --database=#{@config.database} #{files.join ' '}"
       cmd += " | #{@config.bin_path}mysql -u'#{@config.user}'"
       cmd += " -p'#{@config.password}'" if @config.password
       run cmd

data/lib/mys3ql/s3.rb

@@ -1,5 +1,5 @@
 require 'mys3ql/shell'
-require 'fog'
+require 'aws-sdk-s3'
 
 module Mys3ql
   class S3
@@ -7,7 +7,6 @@ module Mys3ql
 
     def initialize(config)
       @config = config
-      Fog::Logger[:warning] = nil
     end
 
     def store(file, dump = true)
@@ -15,20 +14,28 @@ module Mys3ql
       s3_file = save file, key
       if dump && s3_file
         copy_key = key_for :latest
-        s3_file.copy @config.bucket, copy_key
+        s3_file.copy_to key: copy_key
         log "s3: copied #{key} to #{copy_key}"
       end
     end
 
     def delete_bin_logs
       each_bin_log do |file|
-        file.destroy
+        file.delete
         log "s3: deleted #{file.key}"
       end
     end
 
-    def each_bin_log(&block)
-      bucket.files.all(:prefix => "#{bin_logs_prefix}").sort_by { |file| file.key[/\d+/].to_i }.each do |file|
+    def each_bin_log(after = nil, &block)
+      if after && after !~ /^\d{6}$/
+        puts 'Binary log file number must be 6 digits.'
+        exit 1
+      end
+
+      bucket.objects(prefix: bin_logs_prefix)
+            .sort_by { |file| file.key[/\d+/].to_i }
+            .select  { |file| after.nil? || (file.key[/\d+/].to_i > after.to_i) }
+            .each do |file|
         yield file
       end
     end
@@ -41,25 +48,28 @@ module Mys3ql
     private
 
     def get(s3_key, local_file_name)
-      s3_file = bucket.files.get s3_key
-      File.open(local_file_name, 'wb') do |file|
-        file.write s3_file.body
-      end
+      s3.get_object(
+        response_target: local_file_name,
+        bucket:          @config.bucket,
+        key:             s3_key
+      )
       log "s3: pulled #{s3_key} to #{local_file_name}"
     end
 
-    # returns Fog::Storage::AWS::File if we pushed, nil otherwise.
     def save(local_file_name, s3_key)
-      s3.sync_clock
-      unless bucket.files.head(s3_key)
-        s3_file = bucket.files.create(
-          :key    => s3_key,
-          :body   => File.open(local_file_name),
-          :public => false
-        )
-        log "s3: pushed #{local_file_name} to #{s3_key}"
-        s3_file
+      if bucket.object(s3_key).exists?
+        log "s3: skipped #{local_file_name} - already exists"
+        return
       end
+
+      s3_file = bucket.put_object(
+        key:           s3_key,
+        body:          File.open(local_file_name),
+        storage_class: 'STANDARD_IA',
+        acl:           'private'
+      )
+      log "s3: pushed #{local_file_name} to #{s3_key}"
+      s3_file
     end
 
     def key_for(kind, file = nil)
@@ -73,23 +83,22 @@ module Mys3ql
 
     def s3
       @s3 ||= begin
-        s = Fog::Storage.new(
-          :provider              => 'AWS',
-          :aws_secret_access_key => @config.secret_access_key,
-          :aws_access_key_id     => @config.access_key_id,
-          :region                => @config.region
+        client = Aws::S3::Client.new(
+          secret_access_key: @config.secret_access_key,
+          access_key_id:     @config.access_key_id,
+          region:            @config.region
         )
         log 's3: connected'
-        s
+        client
       end
     end
 
     def bucket
-      @directory ||= begin
-        d = s3.directories.get @config.bucket
-        raise "S3 bucket #{@config.bucket} not found" unless d  # create bucket instead (n.b. region/location)?
+      @bucket ||= begin
+        b = Aws::S3::Bucket.new @config.bucket, client: s3
+        raise "S3 bucket #{@config.bucket} not found" unless b.exists?
         log "s3: opened bucket #{@config.bucket}"
-        d
+        b
       end
     end
 
@@ -104,6 +113,5 @@ module Mys3ql
     def bin_logs_exist?
       @config.bin_log && @config.bin_log.length > 0 && File.exist?(@config.bin_log)
     end
-
   end
 end

data/lib/mys3ql/shell.rb

@@ -1,5 +1,5 @@
 module Mys3ql
-  class ShellCommandError < RuntimeError ; end
+  ShellCommandError = Class.new RuntimeError
 
   module Shell
     def run(command)

data/lib/mys3ql/version.rb

@@ -1,3 +1,3 @@
 module Mys3ql
-  VERSION = '1.0.2'
+  VERSION = '1.3.0'
 end

data/mys3ql.gemspec

@@ -11,14 +11,11 @@ Gem::Specification.new do |s|
   s.summary     = 'Simple backup of your MySql database onto Amazon S3.'
   s.description = s.summary
 
-  s.rubyforge_project = "mys3ql"
-
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency 'main', '~> 4.8.0'
-  s.add_dependency 'fog', '~> 1.19.0'
+  s.add_dependency 'aws-sdk-s3', '~> 1'
   s.add_development_dependency 'rake'
 end

metadata

@@ -1,62 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: mys3ql
 version: !ruby/object:Gem::Version
-  version: 1.0.2
-  prerelease: 
+  version: 1.3.0
 platform: ruby
 authors:
 - Andy Stewart
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-01-07 00:00:00.000000000 Z
+date: 2021-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: main
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.8.0
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.8.0
-- !ruby/object:Gem::Dependency
-  name: fog
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.19.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.19.0
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Simple backup of your MySql database onto Amazon S3.
@@ -67,8 +46,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- CHANGELOG.md
+- ".gitignore"
 - Gemfile
 - README.md
 - Rakefile
@@ -83,32 +61,25 @@ files:
 - mys3ql.gemspec
 homepage: https://github.com/airblade/mys3ql
 licenses: []
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 271295663072223621
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 271295663072223621
 requirements: []
-rubyforge_project: mys3ql
-rubygems_version: 1.8.23
-signing_key: 
-specification_version: 3
+rubyforge_project:
+rubygems_version: 2.5.2.3
+signing_key:
+specification_version: 4
 summary: Simple backup of your MySql database onto Amazon S3.
 test_files: []

data/CHANGELOG.md

@@ -1,3 +0,0 @@
-# Changelog
-
-[awaiting v1.0.0]