mys3ql 1.0.0 → 1.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5c654cea5796fd8510c97cadadfd2eec223711f0b9c67c71b6050af4a79e7223
+  data.tar.gz: 621782acd4a75e1be1c292621bae70aeecfc758a95a364de5201ed5f42c1b79a
+SHA512:
+  metadata.gz: 7288bf4d3b1f9a208d19486ca5e7d8515fb3a97b1a66dee603f44c4571e8937c1a743a66963610f3dd744136aaa7fa506b44171c94bb9272253c69e09e50aa2c
+  data.tar.gz: 01a4dc81218fd10dd69f4770e6aeb933e634360033d6adcb85ca4a48528e8c101e60f25ec29223506b96dfb321f91cc5fdbe7999e12fde8458f5a22fbbb9c6cb

data/README.md

@@ -2,6 +2,8 @@
 
 Simple backup of your MySQL database onto Amazon S3.
 
+See [Example: mysqldump + mysqlbinlog for Backup and Restore](https://dev.mysql.com/doc/refman/5.7/en/mysqlbinlog-backup.html#mysqlbinlog-backup-example).
+
 
 ## Quick start
 
@@ -52,6 +54,9 @@ Second, create your config file:
       secret_access_key: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
       # Bucket in which to store your backups
       bucket: db_backups
+      # AWS region your bucket lives in.
+      # (I suspect you only need to specify this when your 'location' is in a different region.)
+      #region: eu-west-1
 
 If you only have one database to back up on your server, you can put the config file at `~/.mys3ql`.  Otherwise, tell the `mys3ql` command where the config file is with the `--config=FILE` switch.
 
@@ -76,20 +81,8 @@ N.B. the binary logs contain updates to all the databases on the server.  This m
 Marc-André Cournoyer's [mysql_s3_backup](https://github.com/macournoyer/mysql_s3_backup).
 
 
-## To Do
-
-- tests ;)
-- remove old dump files (s3)
-- (restore from non-latest dump)
-
-
-## Questions, Problems, Feedback
-
-Please use the GitHub [issue tracker](https://github.com/airblade/mys3ql/issues) or email me.
-
-
 ## Intellectual property
 
-Copyright 2011 Andy Stewart (boss@airbladesoftware.com).
+Copyright 2011-2021 Andy Stewart (boss@airbladesoftware.com).
 
 Released under the MIT licence.

data/bin/mys3ql

@@ -1,35 +1,26 @@
-#!/usr/bin/env ruby-local-exec
-
-lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
-$LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
+#!/usr/bin/env ruby
 
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'mys3ql'
-require 'main'
-
-Main do
-  # consider using modes as/when we need command-specific arguments (e.g. restore specific backup)
-  argument 'command' do
-    validate { |command| %w[ full incremental restore ].include? command }
-    description 'specifies the operation to perform [full | incremental | restore]'
-  end
+require 'optparse'
 
-  option 'config', 'c' do
-    argument :required
-    description 'load configuration from YAML file'
-    defaults '~/.mys3ql'
-  end
+params = {}
+op = OptionParser.new do |opts|
+  opts.banner = 'Usage: mys3ql <full|incremental|restore> <[args]>'
 
-  option 'debug', 'd' do
-    description 'be verbose'
+  opts.on '-c', '--config CONFIG', 'Load configuration from YAML file (default ~/.mys3ql)'
+  opts.on_tail '-d', '--debug', 'Be verbose'
+  opts.on_tail '-v', '--version', 'Print version' do
+    puts "mys3ql v#{Mys3ql::VERSION}"
+    exit
   end
+end
+op.parse!(into: params)
 
-  option 'version', 'v'
-
-  def run
-    if params[:version].given?
-      puts "mys3ql v#{Mys3ql::VERSION}"
-      exit
-    end
-    Mys3ql::Conductor.run params[:command].value, params[:config].value, params[:debug].given?
-  end
+params[:command] = ARGV[0]
+unless %w[full incremental restore].include? params[:command]
+  puts op.help
+  exit 1
 end
+
+Mys3ql::Conductor.run params[:command], params[:config], params[:debug]

data/lib/mys3ql/conductor.rb

@@ -18,6 +18,10 @@ module Mys3ql
       @s3 = S3.new @config
     end
 
+    # Dumps the database and uploads it to S3.
+    # Copies the uploaded file to the key :latest.
+    # Deletes binary logs from the file system.
+    # Deletes binary logs from S3.
     def full
       @mysql.dump
       @s3.store @mysql.dump_file
@@ -25,13 +29,18 @@ module Mys3ql
       @s3.delete_bin_logs
     end
 
+    # Uploads mysql's binary logs to S3.
+    # The binary logs are left on the file system.
+    # Log files already on S3 are not re-uploaded.
     def incremental
       @mysql.each_bin_log do |log|
         @s3.store log, false
       end
     end
 
-    # for now only restore from latest
+    # Downloads the latest dump from S3 and loads it into the database.
+    # Downloads each binary log from S3 and loads it into the database.
+    # Downloaded files are removed from the file system.
     def restore
       # get latest dump
       with_temp_file do |file|
@@ -48,7 +57,7 @@ module Mys3ql
       end
 
       # NOTE: not sure about this:
-      puts "You might want to flush mysql's logs..."
+      # puts "You might want to flush mysql's logs..."
     end
 
     def debug=(val)

data/lib/mys3ql/config.rb

@@ -63,6 +63,10 @@ module Mys3ql
       s3['bucket']
     end
 
+    def region
+      s3['region']
+    end
+
     private
 
     def mysql

data/lib/mys3ql/mysql.rb

@@ -13,8 +13,10 @@ module Mys3ql
     #
 
     def dump
+      # --master-data=2         include the current binary log coordinates in the log file
+      # --delete-master-logs    delete binary log files
       cmd  = "#{@config.bin_path}mysqldump"
-      cmd += ' --quick --single-transaction --create-options'
+      cmd += ' --quick --single-transaction --create-options --no-tablespaces'
       cmd += ' --flush-logs --master-data=2 --delete-master-logs' if binary_logging?
       cmd += cli_options
       cmd += " | gzip > #{dump_file}"
@@ -36,14 +38,18 @@ module Mys3ql
 
     # flushes logs, yields each bar the last to the block
     def each_bin_log(&block)
+      # FLUSH LOGS    Closes and reopens any log file, including binary logs,
+      #               to which the server is writing.  For binary logs, the sequence
+      #               number of the binary log file is incremented by one relative to
+      #               the previous file.
+      #               https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-logs
+      #               https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-binary-logs
       execute 'flush logs'
       logs = Dir.glob("#{@config.bin_log}.[0-9]*").sort_by { |f| f[/\d+/].to_i }
-      logs_to_backup = logs[0..-2]  # all logs except the last, which is in use
+      logs_to_backup = logs[0..-2]  # all logs except the last, which is newly created
       logs_to_backup.each do |log_file|
         yield log_file
       end
-      # delete binlogs from file system
-      #execute "purge master logs to '#{File.basename(logs[-1])}'"
     end
 
     #

data/lib/mys3ql/s3.rb

@@ -1,5 +1,5 @@
 require 'mys3ql/shell'
-require 'fog'
+require 'awd-sdk-s3'
 
 module Mys3ql
   class S3
@@ -7,7 +7,6 @@ module Mys3ql
 
     def initialize(config)
       @config = config
-      Fog::Logger[:warning] = nil
     end
 
     def store(file, dump = true)
@@ -15,20 +14,22 @@ module Mys3ql
       s3_file = save file, key
       if dump && s3_file
         copy_key = key_for :latest
-        s3_file.copy @config.bucket, copy_key
+        s3_file.copy_to key: copy_key
         log "s3: copied #{key} to #{copy_key}"
       end
     end
 
     def delete_bin_logs
       each_bin_log do |file|
-        file.destroy
+        file.delete
         log "s3: deleted #{file.key}"
       end
     end
 
     def each_bin_log(&block)
-      bucket.files.all(:prefix => "#{bin_logs_prefix}").sort_by { |file| file.key[/\d+/].to_i }.each do |file|
+      bucket.objects(prefix: bin_logs_prefix)
+            .sort_by { |file| file.key[/\d+/].to_i }
+            .each do |file|
         yield file
       end
     end
@@ -41,24 +42,28 @@ module Mys3ql
     private
 
     def get(s3_key, local_file_name)
-      s3_file = bucket.files.get s3_key
-      File.open(local_file_name, 'wb') do |file|
-        file.write s3_file.body
-      end
+      s3.get_object(
+        response_target: local_file_name,
+        bucket:          @config.bucket,
+        key:             s3_key
+      )
       log "s3: pulled #{s3_key} to #{local_file_name}"
     end
 
-    # returns Fog::Storage::AWS::File if we pushed, nil otherwise.
     def save(local_file_name, s3_key)
-      unless bucket.files.head(s3_key)
-        s3_file = bucket.files.create(
-          :key    => s3_key,
-          :body   => File.open(local_file_name),
-          :public => false
-        )
-        log "s3: pushed #{local_file_name} to #{s3_key}"
-        s3_file
+      if bucket.object(s3_key).exists?
+        log "s3: skipped #{local_file_name} - #{s3_key} exists"
+        return
       end
+
+      s3_file = bucket.put_object(
+        key:           s3_key,
+        body:          File.open(local_file_name),
+        storage_class: 'STANDARD_IA',
+        acl:           'private'
+      )
+      log "s3: pushed #{local_file_name} to #{s3_key}"
+      s3_file
     end
 
     def key_for(kind, file = nil)
@@ -72,22 +77,22 @@ module Mys3ql
 
     def s3
       @s3 ||= begin
-        s = Fog::Storage.new(
-          :provider              => 'AWS',
-          :aws_secret_access_key => @config.secret_access_key,
-          :aws_access_key_id     => @config.access_key_id
+        client = Aws::S3::Client.new(
+          secret_access_key: @config.secret_access_key,
+          access_key_id:     @config.access_key_id,
+          region:            @config.region
         )
         log 's3: connected'
-        s
+        client
       end
     end
 
     def bucket
-      @directory ||= begin
-        d = s3.directories.get @config.bucket
-        raise "S3 bucket #{@config.bucket} not found" unless d  # create bucket instead?
+      @bucket ||= begin
+        b = Aws::S3::Bucket.new @config.bucket, client: s3
+        raise "S3 bucket #{@config.bucket} not found" unless b.exists?
         log "s3: opened bucket #{@config.bucket}"
-        d
+        b
       end
     end
 
@@ -102,6 +107,5 @@ module Mys3ql
     def bin_logs_exist?
       @config.bin_log && @config.bin_log.length > 0 && File.exist?(@config.bin_log)
     end
-
   end
 end

data/lib/mys3ql/shell.rb

@@ -1,5 +1,5 @@
 module Mys3ql
-  class ShellCommandError < RuntimeError ; end
+  ShellCommandError = Class.new RuntimeError
 
   module Shell
     def run(command)

data/lib/mys3ql/version.rb

@@ -1,3 +1,3 @@
 module Mys3ql
-  VERSION = '1.0.0'
+  VERSION = '1.2.0'
 end

data/mys3ql.gemspec

@@ -11,14 +11,11 @@ Gem::Specification.new do |s|
   s.summary     = 'Simple backup of your MySql database onto Amazon S3.'
   s.description = s.summary
 
-  s.rubyforge_project = "mys3ql"
-
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency 'main', '~> 4.8.0'
-  s.add_dependency 'fog', '~> 1.0.0'
+  s.add_dependency 'aws-sdk-s3', '~> 1'
   s.add_development_dependency 'rake'
 end

metadata

@@ -1,81 +1,52 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: mys3ql
-version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: 
-  segments: 
-  - 1
-  - 0
-  - 0
-  version: 1.0.0
+version: !ruby/object:Gem::Version
+  version: 1.2.0
 platform: ruby
-authors: 
+authors:
 - Andy Stewart
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-
-date: 2011-11-04 00:00:00 +01:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: main
-  prerelease: false
-  version_requirements: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 31
-        segments: 
-        - 4
-        - 8
-        - 0
-        version: 4.8.0
+date: 2021-02-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
   type: :runtime
-  requirement: *id001
-- !ruby/object:Gem::Dependency 
-  name: fog
   prerelease: false
-  version_requirements: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
-        version: 1.0.0
-  type: :runtime
-  requirement: *id002
-- !ruby/object:Gem::Dependency 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
   name: rake
-  prerelease: false
-  version_requirements: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
     - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  requirement: *id003
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Simple backup of your MySql database onto Amazon S3.
-email: 
+email:
 - boss@airbladesoftware.com
-executables: 
+executables:
 - mys3ql
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- .gitignore
-- CHANGELOG.md
+files:
+- ".gitignore"
 - Gemfile
 - README.md
 - Rakefile
@@ -88,39 +59,26 @@ files:
 - lib/mys3ql/shell.rb
 - lib/mys3ql/version.rb
 - mys3ql.gemspec
-has_rdoc: true
 homepage: https://github.com/airblade/mys3ql
 licenses: []
-
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
-rubyforge_project: mys3ql
-rubygems_version: 1.6.2
-signing_key: 
-specification_version: 3
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
 summary: Simple backup of your MySql database onto Amazon S3.
 test_files: []
-

data/CHANGELOG.md

@@ -1,3 +0,0 @@
-# Changelog
-
-[awaiting v1.0.0]