mys3ql 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e54e18bc02f3e450ff0b3bbcf538d110740f7656
-  data.tar.gz: d9a68227ab366c3cfae81f56116c723c0ccb4bb4
+SHA256:
+  metadata.gz: 5c654cea5796fd8510c97cadadfd2eec223711f0b9c67c71b6050af4a79e7223
+  data.tar.gz: 621782acd4a75e1be1c292621bae70aeecfc758a95a364de5201ed5f42c1b79a
 SHA512:
-  metadata.gz: cec4e35514acad343828bfbd6c726db0a0ca3654ce54906e1cb047d242a68d3b4fb7c47bde63287b8fda3a5fe6a0c01ecec19954f92be10a523ea954fce24261
-  data.tar.gz: e62c4784ea4161dd939f659ffa8f80d765976059149185adbf724d9eb4ffce6e2445296051cba0040d6369f9a24a028d8a0185bd12f66d4d16d259b17ac12d31
+  metadata.gz: 7288bf4d3b1f9a208d19486ca5e7d8515fb3a97b1a66dee603f44c4571e8937c1a743a66963610f3dd744136aaa7fa506b44171c94bb9272253c69e09e50aa2c
+  data.tar.gz: 01a4dc81218fd10dd69f4770e6aeb933e634360033d6adcb85ca4a48528e8c101e60f25ec29223506b96dfb321f91cc5fdbe7999e12fde8458f5a22fbbb9c6cb

data/README.md

@@ -2,6 +2,8 @@
 
 Simple backup of your MySQL database onto Amazon S3.
 
+See [Example: mysqldump + mysqlbinlog for Backup and Restore](https://dev.mysql.com/doc/refman/5.7/en/mysqlbinlog-backup.html#mysqlbinlog-backup-example).
+
 
 ## Quick start
 
@@ -79,20 +81,8 @@ N.B. the binary logs contain updates to all the databases on the server.  This m
 Marc-André Cournoyer's [mysql_s3_backup](https://github.com/macournoyer/mysql_s3_backup).
 
 
-## To Do
-
-- tests ;)
-- remove old dump files (s3)
-- (restore from non-latest dump)
-
-
-## Questions, Problems, Feedback
-
-Please use the GitHub [issue tracker](https://github.com/airblade/mys3ql/issues) or email me.
-
-
 ## Intellectual property
 
-Copyright 2011 Andy Stewart (boss@airbladesoftware.com).
+Copyright 2011-2021 Andy Stewart (boss@airbladesoftware.com).
 
 Released under the MIT licence.

data/bin/mys3ql

@@ -1,35 +1,26 @@
-#!/usr/bin/env ruby-local-exec
-
-lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
-$LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
+#!/usr/bin/env ruby
 
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'mys3ql'
-require 'main'
-
-Main do
-  # consider using modes as/when we need command-specific arguments (e.g. restore specific backup)
-  argument 'command' do
-    validate { |command| %w[ full incremental restore ].include? command }
-    description 'specifies the operation to perform [full | incremental | restore]'
-  end
+require 'optparse'
 
-  option 'config', 'c' do
-    argument :required
-    description 'load configuration from YAML file'
-    defaults '~/.mys3ql'
-  end
+params = {}
+op = OptionParser.new do |opts|
+  opts.banner = 'Usage: mys3ql <full|incremental|restore> <[args]>'
 
-  option 'debug', 'd' do
-    description 'be verbose'
+  opts.on '-c', '--config CONFIG', 'Load configuration from YAML file (default ~/.mys3ql)'
+  opts.on_tail '-d', '--debug', 'Be verbose'
+  opts.on_tail '-v', '--version', 'Print version' do
+    puts "mys3ql v#{Mys3ql::VERSION}"
+    exit
   end
+end
+op.parse!(into: params)
 
-  option 'version', 'v'
-
-  def run
-    if params[:version].given?
-      puts "mys3ql v#{Mys3ql::VERSION}"
-      exit
-    end
-    Mys3ql::Conductor.run params[:command].value, params[:config].value, params[:debug].given?
-  end
+params[:command] = ARGV[0]
+unless %w[full incremental restore].include? params[:command]
+  puts op.help
+  exit 1
 end
+
+Mys3ql::Conductor.run params[:command], params[:config], params[:debug]

data/lib/mys3ql/conductor.rb

@@ -18,6 +18,10 @@ module Mys3ql
       @s3 = S3.new @config
     end
 
+    # Dumps the database and uploads it to S3.
+    # Copies the uploaded file to the key :latest.
+    # Deletes binary logs from the file system.
+    # Deletes binary logs from S3.
     def full
       @mysql.dump
       @s3.store @mysql.dump_file
@@ -25,13 +29,18 @@ module Mys3ql
       @s3.delete_bin_logs
     end
 
+    # Uploads mysql's binary logs to S3.
+    # The binary logs are left on the file system.
+    # Log files already on S3 are not re-uploaded.
     def incremental
       @mysql.each_bin_log do |log|
         @s3.store log, false
       end
     end
 
-    # for now only restore from latest
+    # Downloads the latest dump from S3 and loads it into the database.
+    # Downloads each binary log from S3 and loads it into the database.
+    # Downloaded files are removed from the file system.
     def restore
       # get latest dump
       with_temp_file do |file|
@@ -48,7 +57,7 @@ module Mys3ql
       end
 
       # NOTE: not sure about this:
-      puts "You might want to flush mysql's logs..."
+      # puts "You might want to flush mysql's logs..."
     end
 
     def debug=(val)

data/lib/mys3ql/mysql.rb

@@ -13,6 +13,8 @@ module Mys3ql
     #
 
     def dump
+      # --master-data=2         include the current binary log coordinates in the log file
+      # --delete-master-logs    delete binary log files
       cmd  = "#{@config.bin_path}mysqldump"
       cmd += ' --quick --single-transaction --create-options --no-tablespaces'
       cmd += ' --flush-logs --master-data=2 --delete-master-logs' if binary_logging?
@@ -36,14 +38,18 @@ module Mys3ql
 
     # flushes logs, yields each bar the last to the block
     def each_bin_log(&block)
+      # FLUSH LOGS    Closes and reopens any log file, including binary logs,
+      #               to which the server is writing.  For binary logs, the sequence
+      #               number of the binary log file is incremented by one relative to
+      #               the previous file.
+      #               https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-logs
+      #               https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-binary-logs
       execute 'flush logs'
       logs = Dir.glob("#{@config.bin_log}.[0-9]*").sort_by { |f| f[/\d+/].to_i }
-      logs_to_backup = logs[0..-2]  # all logs except the last, which is in use
+      logs_to_backup = logs[0..-2]  # all logs except the last, which is newly created
       logs_to_backup.each do |log_file|
         yield log_file
       end
-      # delete binlogs from file system
-      #execute "purge master logs to '#{File.basename(logs[-1])}'"
     end
 
     #

data/lib/mys3ql/s3.rb

@@ -1,5 +1,5 @@
 require 'mys3ql/shell'
-require 'fog'
+require 'awd-sdk-s3'
 
 module Mys3ql
   class S3
@@ -7,7 +7,6 @@ module Mys3ql
 
     def initialize(config)
       @config = config
-      Fog::Logger[:warning] = nil
     end
 
     def store(file, dump = true)
@@ -15,20 +14,22 @@ module Mys3ql
       s3_file = save file, key
       if dump && s3_file
         copy_key = key_for :latest
-        s3_file.copy @config.bucket, copy_key
+        s3_file.copy_to key: copy_key
         log "s3: copied #{key} to #{copy_key}"
       end
     end
 
     def delete_bin_logs
       each_bin_log do |file|
-        file.destroy
+        file.delete
         log "s3: deleted #{file.key}"
       end
     end
 
     def each_bin_log(&block)
-      bucket.files.all(:prefix => "#{bin_logs_prefix}").sort_by { |file| file.key[/\d+/].to_i }.each do |file|
+      bucket.objects(prefix: bin_logs_prefix)
+            .sort_by { |file| file.key[/\d+/].to_i }
+            .each do |file|
         yield file
       end
     end
@@ -41,26 +42,28 @@ module Mys3ql
     private
 
     def get(s3_key, local_file_name)
-      s3_file = bucket.files.get s3_key
-      File.open(local_file_name, 'wb') do |file|
-        file.write s3_file.body
-      end
+      s3.get_object(
+        response_target: local_file_name,
+        bucket:          @config.bucket,
+        key:             s3_key
+      )
       log "s3: pulled #{s3_key} to #{local_file_name}"
     end
 
-    # returns Fog::Storage::AWS::File if we pushed, nil otherwise.
     def save(local_file_name, s3_key)
-      s3.sync_clock
-      unless bucket.files.head(s3_key)
-        s3_file = bucket.files.create(
-          :key    => s3_key,
-          :body   => File.open(local_file_name),
-          :storage_class => 'STANDARD_IA',
-          :public => false
-        )
-        log "s3: pushed #{local_file_name} to #{s3_key}"
-        s3_file
+      if bucket.object(s3_key).exists?
+        log "s3: skipped #{local_file_name} - #{s3_key} exists"
+        return
       end
+
+      s3_file = bucket.put_object(
+        key:           s3_key,
+        body:          File.open(local_file_name),
+        storage_class: 'STANDARD_IA',
+        acl:           'private'
+      )
+      log "s3: pushed #{local_file_name} to #{s3_key}"
+      s3_file
     end
 
     def key_for(kind, file = nil)
@@ -74,23 +77,22 @@ module Mys3ql
 
     def s3
       @s3 ||= begin
-        s = Fog::Storage.new(
-          :provider              => 'AWS',
-          :aws_secret_access_key => @config.secret_access_key,
-          :aws_access_key_id     => @config.access_key_id,
-          :region                => @config.region
+        client = Aws::S3::Client.new(
+          secret_access_key: @config.secret_access_key,
+          access_key_id:     @config.access_key_id,
+          region:            @config.region
         )
         log 's3: connected'
-        s
+        client
       end
     end
 
     def bucket
-      @directory ||= begin
-        d = s3.directories.get @config.bucket
-        raise "S3 bucket #{@config.bucket} not found" unless d  # create bucket instead (n.b. region/location)?
+      @bucket ||= begin
+        b = Aws::S3::Bucket.new @config.bucket, client: s3
+        raise "S3 bucket #{@config.bucket} not found" unless b.exists?
         log "s3: opened bucket #{@config.bucket}"
-        d
+        b
       end
     end
 
@@ -105,6 +107,5 @@ module Mys3ql
     def bin_logs_exist?
       @config.bin_log && @config.bin_log.length > 0 && File.exist?(@config.bin_log)
     end
-
   end
 end

data/lib/mys3ql/shell.rb

@@ -1,5 +1,5 @@
 module Mys3ql
-  class ShellCommandError < RuntimeError ; end
+  ShellCommandError = Class.new RuntimeError
 
   module Shell
     def run(command)

data/lib/mys3ql/version.rb

@@ -1,3 +1,3 @@
 module Mys3ql
-  VERSION = '1.1.0'
+  VERSION = '1.2.0'
 end

data/mys3ql.gemspec

@@ -11,14 +11,11 @@ Gem::Specification.new do |s|
   s.summary     = 'Simple backup of your MySql database onto Amazon S3.'
   s.description = s.summary
 
-  s.rubyforge_project = "mys3ql"
-
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency 'main', '~> 4.8.0'
-  s.add_dependency 'fog', '~> 1.19.0'
+  s.add_dependency 'aws-sdk-s3', '~> 1'
   s.add_development_dependency 'rake'
 end

metadata

@@ -1,43 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: mys3ql
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Andy Stewart
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-22 00:00:00.000000000 Z
+date: 2021-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: main
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.8.0
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.8.0
-- !ruby/object:Gem::Dependency
-  name: fog
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.19.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.19.0
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -61,7 +47,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- CHANGELOG.md
 - Gemfile
 - README.md
 - Rakefile
@@ -92,8 +77,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: mys3ql
-rubygems_version: 2.5.2.3
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: Simple backup of your MySql database onto Amazon S3.

data/CHANGELOG.md

@@ -1,3 +0,0 @@
-# Changelog
-
-[awaiting v1.0.0]