mys3ql 1.0.2 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +9 -13
- data/bin/mys3ql +24 -26
- data/lib/mys3ql/conductor.rb +45 -16
- data/lib/mys3ql/mysql.rb +12 -6
- data/lib/mys3ql/s3.rb +39 -31
- data/lib/mys3ql/shell.rb +1 -1
- data/lib/mys3ql/version.rb +1 -1
- data/mys3ql.gemspec +1 -4
- metadata +19 -48
- data/CHANGELOG.md +0 -3
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 86d0a66ce7dab918b298c22eb4d667df2cb9ac7c
|
4
|
+
data.tar.gz: cd9527db20ae5a07349b3456e544a0aa3552f9bd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 25abe9f3cc7daee801c8402dbd07b8827608d139ed4a2ca95ec089ed1a7d4335dfbfaa8fd30f10ea06118159d3513fa90c866ac9fa09976b48dc8d05645ac105
|
7
|
+
data.tar.gz: 340d29d1230eda51d4408b8f5fb3597ed7da31276171608469092c10fbe04ceb5f3e0640f3758cbdbe14c2af99804bc5ebb81dd27e6097c661c492d4339d56f4
|
data/README.md
CHANGED
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
Simple backup of your MySQL database onto Amazon S3.
|
4
4
|
|
5
|
+
See [Example: mysqldump + mysqlbinlog for Backup and Restore](https://dev.mysql.com/doc/refman/5.7/en/mysqlbinlog-backup.html#mysqlbinlog-backup-example).
|
6
|
+
|
5
7
|
|
6
8
|
## Quick start
|
7
9
|
|
@@ -19,6 +21,12 @@ To restore from the latest backup (plus binlogs if present):
|
|
19
21
|
|
20
22
|
$ mys3ql restore
|
21
23
|
|
24
|
+
To restore a recent subset of binlogs:
|
25
|
+
|
26
|
+
$ mys3ql restore --after NUMBER
|
27
|
+
|
28
|
+
– where NUMBER is a 6-digit binlog file number.
|
29
|
+
|
22
30
|
By default mys3ql looks for a configuration file at `~/.mys3ql`. You can override this like so:
|
23
31
|
|
24
32
|
$ mys3ql [command] -c FILE
|
@@ -79,20 +87,8 @@ N.B. the binary logs contain updates to all the databases on the server. This m
|
|
79
87
|
Marc-André Cournoyer's [mysql_s3_backup](https://github.com/macournoyer/mysql_s3_backup).
|
80
88
|
|
81
89
|
|
82
|
-
## To Do
|
83
|
-
|
84
|
-
- tests ;)
|
85
|
-
- remove old dump files (s3)
|
86
|
-
- (restore from non-latest dump)
|
87
|
-
|
88
|
-
|
89
|
-
## Questions, Problems, Feedback
|
90
|
-
|
91
|
-
Please use the GitHub [issue tracker](https://github.com/airblade/mys3ql/issues) or email me.
|
92
|
-
|
93
|
-
|
94
90
|
## Intellectual property
|
95
91
|
|
96
|
-
Copyright 2011 Andy Stewart (boss@airbladesoftware.com).
|
92
|
+
Copyright 2011-2021 Andy Stewart (boss@airbladesoftware.com).
|
97
93
|
|
98
94
|
Released under the MIT licence.
|
data/bin/mys3ql
CHANGED
@@ -1,35 +1,33 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
lib_dir = File.join(File.dirname(__FILE__), '..', 'lib')
|
4
|
-
$LOAD_PATH.unshift lib_dir if File.directory?(lib_dir)
|
1
|
+
#!/usr/bin/env ruby
|
5
2
|
|
3
|
+
$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
|
6
4
|
require 'mys3ql'
|
7
|
-
require '
|
5
|
+
require 'optparse'
|
8
6
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
validate { |command| %w[ full incremental restore ].include? command }
|
13
|
-
description 'specifies the operation to perform [full | incremental | restore]'
|
14
|
-
end
|
7
|
+
params = {}
|
8
|
+
op = OptionParser.new do |opts|
|
9
|
+
opts.banner = 'Usage: mys3ql <full|incremental|restore> [<options>]'
|
15
10
|
|
16
|
-
|
17
|
-
|
18
|
-
description 'load configuration from YAML file'
|
19
|
-
defaults '~/.mys3ql'
|
20
|
-
end
|
11
|
+
opts.separator ''
|
12
|
+
opts.separator 'Common options:'
|
21
13
|
|
22
|
-
|
23
|
-
|
14
|
+
opts.on('-c', '--config CONFIG', 'Load configuration from YAML file (default ~/.mys3ql)') { |v| params[:config] = v }
|
15
|
+
opts.on('-d', '--debug', 'Be verbose') { |v| params[:debug] = v }
|
16
|
+
opts.on '-v', '--version', 'Print version' do
|
17
|
+
puts "mys3ql v#{Mys3ql::VERSION}"
|
18
|
+
exit
|
24
19
|
end
|
25
20
|
|
26
|
-
|
21
|
+
opts.separator ''
|
22
|
+
opts.separator 'restore options:'
|
23
|
+
opts.on('-a', '--after NUMBER', 'Use only the subset of binary logs after NUMBER') { |v| params[:after] = v }
|
24
|
+
end
|
25
|
+
op.parse! ARGV
|
27
26
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
end
|
33
|
-
Mys3ql::Conductor.run params[:command].value, params[:config].value, params[:debug].given?
|
34
|
-
end
|
27
|
+
params[:command] = ARGV[0]
|
28
|
+
unless %w[full incremental restore].include? params[:command]
|
29
|
+
puts op.help
|
30
|
+
exit 1
|
35
31
|
end
|
32
|
+
|
33
|
+
Mys3ql::Conductor.run params
|
data/lib/mys3ql/conductor.rb
CHANGED
@@ -6,10 +6,16 @@ require 'mys3ql/s3'
|
|
6
6
|
module Mys3ql
|
7
7
|
class Conductor
|
8
8
|
|
9
|
-
def self.run(
|
10
|
-
conductor = Conductor.new(config)
|
11
|
-
conductor.debug = debug
|
12
|
-
|
9
|
+
def self.run(args)
|
10
|
+
conductor = Conductor.new args.fetch(:config, nil)
|
11
|
+
conductor.debug = args.fetch(:debug, false)
|
12
|
+
|
13
|
+
command = args.fetch(:command)
|
14
|
+
if command == 'restore'
|
15
|
+
conductor.restore args.fetch(:after, nil)
|
16
|
+
else
|
17
|
+
conductor.send command
|
18
|
+
end
|
13
19
|
end
|
14
20
|
|
15
21
|
def initialize(config_file = nil)
|
@@ -18,6 +24,10 @@ module Mys3ql
|
|
18
24
|
@s3 = S3.new @config
|
19
25
|
end
|
20
26
|
|
27
|
+
# Dumps the database and uploads it to S3.
|
28
|
+
# Copies the uploaded file to the key :latest.
|
29
|
+
# Deletes binary logs from the file system.
|
30
|
+
# Deletes binary logs from S3.
|
21
31
|
def full
|
22
32
|
@mysql.dump
|
23
33
|
@s3.store @mysql.dump_file
|
@@ -25,30 +35,49 @@ module Mys3ql
|
|
25
35
|
@s3.delete_bin_logs
|
26
36
|
end
|
27
37
|
|
38
|
+
# Uploads mysql's binary logs to S3.
|
39
|
+
# The binary logs are left on the file system.
|
40
|
+
# Log files already on S3 are not re-uploaded.
|
28
41
|
def incremental
|
29
42
|
@mysql.each_bin_log do |log|
|
30
43
|
@s3.store log, false
|
31
44
|
end
|
32
45
|
end
|
33
46
|
|
34
|
-
#
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
47
|
+
# When 'after' is nil:
|
48
|
+
#
|
49
|
+
# - downloads the latest dump from S3 and loads it into the database;
|
50
|
+
# - downloads each binary log from S3 and loads it into the database.
|
51
|
+
#
|
52
|
+
# When 'after' is given:
|
53
|
+
#
|
54
|
+
# - downloads each binary log following 'after' from S3 and loads it into the database.
|
55
|
+
#
|
56
|
+
# Downloaded files are removed from the file system.
|
57
|
+
def restore(after = nil)
|
58
|
+
unless after
|
59
|
+
# get latest dump
|
60
|
+
with_temp_file do |file|
|
61
|
+
@s3.retrieve :latest, file
|
62
|
+
@mysql.restore file
|
63
|
+
end
|
40
64
|
end
|
41
65
|
|
42
|
-
# apply
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
66
|
+
# apply bin logs
|
67
|
+
begin
|
68
|
+
tmpfiles = []
|
69
|
+
@s3.each_bin_log(after) do |log|
|
70
|
+
file = Tempfile.new 'mys3ql'
|
71
|
+
tmpfiles << file
|
72
|
+
@s3.retrieve log, file.path
|
47
73
|
end
|
74
|
+
@mysql.apply_bin_logs tmpfiles.map(&:path)
|
75
|
+
ensure
|
76
|
+
tmpfiles.each &:close!
|
48
77
|
end
|
49
78
|
|
50
79
|
# NOTE: not sure about this:
|
51
|
-
puts "You might want to flush mysql's logs..."
|
80
|
+
# puts "You might want to flush mysql's logs..."
|
52
81
|
end
|
53
82
|
|
54
83
|
def debug=(val)
|
data/lib/mys3ql/mysql.rb
CHANGED
@@ -13,8 +13,10 @@ module Mys3ql
|
|
13
13
|
#
|
14
14
|
|
15
15
|
def dump
|
16
|
+
# --master-data=2 include the current binary log coordinates in the log file
|
17
|
+
# --delete-master-logs delete binary log files
|
16
18
|
cmd = "#{@config.bin_path}mysqldump"
|
17
|
-
cmd += ' --quick --single-transaction --create-options'
|
19
|
+
cmd += ' --quick --single-transaction --create-options --no-tablespaces'
|
18
20
|
cmd += ' --flush-logs --master-data=2 --delete-master-logs' if binary_logging?
|
19
21
|
cmd += cli_options
|
20
22
|
cmd += " | gzip > #{dump_file}"
|
@@ -36,14 +38,18 @@ module Mys3ql
|
|
36
38
|
|
37
39
|
# flushes logs, yields each bar the last to the block
|
38
40
|
def each_bin_log(&block)
|
41
|
+
# FLUSH LOGS Closes and reopens any log file, including binary logs,
|
42
|
+
# to which the server is writing. For binary logs, the sequence
|
43
|
+
# number of the binary log file is incremented by one relative to
|
44
|
+
# the previous file.
|
45
|
+
# https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-logs
|
46
|
+
# https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-binary-logs
|
39
47
|
execute 'flush logs'
|
40
48
|
logs = Dir.glob("#{@config.bin_log}.[0-9]*").sort_by { |f| f[/\d+/].to_i }
|
41
|
-
logs_to_backup = logs[0..-2] # all logs except the last, which is
|
49
|
+
logs_to_backup = logs[0..-2] # all logs except the last, which is newly created
|
42
50
|
logs_to_backup.each do |log_file|
|
43
51
|
yield log_file
|
44
52
|
end
|
45
|
-
# delete binlogs from file system
|
46
|
-
#execute "purge master logs to '#{File.basename(logs[-1])}'"
|
47
53
|
end
|
48
54
|
|
49
55
|
#
|
@@ -54,8 +60,8 @@ module Mys3ql
|
|
54
60
|
run "gunzip -c #{file} | #{@config.bin_path}mysql #{cli_options}"
|
55
61
|
end
|
56
62
|
|
57
|
-
def
|
58
|
-
cmd = "#{@config.bin_path}mysqlbinlog --database=#{@config.database} #{
|
63
|
+
def apply_bin_logs(*files)
|
64
|
+
cmd = "#{@config.bin_path}mysqlbinlog --database=#{@config.database} #{files.join ' '}"
|
59
65
|
cmd += " | #{@config.bin_path}mysql -u'#{@config.user}'"
|
60
66
|
cmd += " -p'#{@config.password}'" if @config.password
|
61
67
|
run cmd
|
data/lib/mys3ql/s3.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'mys3ql/shell'
|
2
|
-
require '
|
2
|
+
require 'aws-sdk-s3'
|
3
3
|
|
4
4
|
module Mys3ql
|
5
5
|
class S3
|
@@ -7,7 +7,6 @@ module Mys3ql
|
|
7
7
|
|
8
8
|
def initialize(config)
|
9
9
|
@config = config
|
10
|
-
Fog::Logger[:warning] = nil
|
11
10
|
end
|
12
11
|
|
13
12
|
def store(file, dump = true)
|
@@ -15,20 +14,28 @@ module Mys3ql
|
|
15
14
|
s3_file = save file, key
|
16
15
|
if dump && s3_file
|
17
16
|
copy_key = key_for :latest
|
18
|
-
s3_file.
|
17
|
+
s3_file.copy_to key: copy_key
|
19
18
|
log "s3: copied #{key} to #{copy_key}"
|
20
19
|
end
|
21
20
|
end
|
22
21
|
|
23
22
|
def delete_bin_logs
|
24
23
|
each_bin_log do |file|
|
25
|
-
file.
|
24
|
+
file.delete
|
26
25
|
log "s3: deleted #{file.key}"
|
27
26
|
end
|
28
27
|
end
|
29
28
|
|
30
|
-
def each_bin_log(&block)
|
31
|
-
|
29
|
+
def each_bin_log(after = nil, &block)
|
30
|
+
if after && after !~ /^\d{6}$/
|
31
|
+
puts 'Binary log file number must be 6 digits.'
|
32
|
+
exit 1
|
33
|
+
end
|
34
|
+
|
35
|
+
bucket.objects(prefix: bin_logs_prefix)
|
36
|
+
.sort_by { |file| file.key[/\d+/].to_i }
|
37
|
+
.select { |file| after.nil? || (file.key[/\d+/].to_i > after.to_i) }
|
38
|
+
.each do |file|
|
32
39
|
yield file
|
33
40
|
end
|
34
41
|
end
|
@@ -41,25 +48,28 @@ module Mys3ql
|
|
41
48
|
private
|
42
49
|
|
43
50
|
def get(s3_key, local_file_name)
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
51
|
+
s3.get_object(
|
52
|
+
response_target: local_file_name,
|
53
|
+
bucket: @config.bucket,
|
54
|
+
key: s3_key
|
55
|
+
)
|
48
56
|
log "s3: pulled #{s3_key} to #{local_file_name}"
|
49
57
|
end
|
50
58
|
|
51
|
-
# returns Fog::Storage::AWS::File if we pushed, nil otherwise.
|
52
59
|
def save(local_file_name, s3_key)
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
:key => s3_key,
|
57
|
-
:body => File.open(local_file_name),
|
58
|
-
:public => false
|
59
|
-
)
|
60
|
-
log "s3: pushed #{local_file_name} to #{s3_key}"
|
61
|
-
s3_file
|
60
|
+
if bucket.object(s3_key).exists?
|
61
|
+
log "s3: skipped #{local_file_name} - already exists"
|
62
|
+
return
|
62
63
|
end
|
64
|
+
|
65
|
+
s3_file = bucket.put_object(
|
66
|
+
key: s3_key,
|
67
|
+
body: File.open(local_file_name),
|
68
|
+
storage_class: 'STANDARD_IA',
|
69
|
+
acl: 'private'
|
70
|
+
)
|
71
|
+
log "s3: pushed #{local_file_name} to #{s3_key}"
|
72
|
+
s3_file
|
63
73
|
end
|
64
74
|
|
65
75
|
def key_for(kind, file = nil)
|
@@ -73,23 +83,22 @@ module Mys3ql
|
|
73
83
|
|
74
84
|
def s3
|
75
85
|
@s3 ||= begin
|
76
|
-
|
77
|
-
:
|
78
|
-
:
|
79
|
-
:
|
80
|
-
:region => @config.region
|
86
|
+
client = Aws::S3::Client.new(
|
87
|
+
secret_access_key: @config.secret_access_key,
|
88
|
+
access_key_id: @config.access_key_id,
|
89
|
+
region: @config.region
|
81
90
|
)
|
82
91
|
log 's3: connected'
|
83
|
-
|
92
|
+
client
|
84
93
|
end
|
85
94
|
end
|
86
95
|
|
87
96
|
def bucket
|
88
|
-
@
|
89
|
-
|
90
|
-
raise "S3 bucket #{@config.bucket} not found" unless
|
97
|
+
@bucket ||= begin
|
98
|
+
b = Aws::S3::Bucket.new @config.bucket, client: s3
|
99
|
+
raise "S3 bucket #{@config.bucket} not found" unless b.exists?
|
91
100
|
log "s3: opened bucket #{@config.bucket}"
|
92
|
-
|
101
|
+
b
|
93
102
|
end
|
94
103
|
end
|
95
104
|
|
@@ -104,6 +113,5 @@ module Mys3ql
|
|
104
113
|
def bin_logs_exist?
|
105
114
|
@config.bin_log && @config.bin_log.length > 0 && File.exist?(@config.bin_log)
|
106
115
|
end
|
107
|
-
|
108
116
|
end
|
109
117
|
end
|
data/lib/mys3ql/shell.rb
CHANGED
data/lib/mys3ql/version.rb
CHANGED
data/mys3ql.gemspec
CHANGED
@@ -11,14 +11,11 @@ Gem::Specification.new do |s|
|
|
11
11
|
s.summary = 'Simple backup of your MySql database onto Amazon S3.'
|
12
12
|
s.description = s.summary
|
13
13
|
|
14
|
-
s.rubyforge_project = "mys3ql"
|
15
|
-
|
16
14
|
s.files = `git ls-files`.split("\n")
|
17
15
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
16
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
17
|
s.require_paths = ["lib"]
|
20
18
|
|
21
|
-
s.add_dependency '
|
22
|
-
s.add_dependency 'fog', '~> 1.19.0'
|
19
|
+
s.add_dependency 'aws-sdk-s3', '~> 1'
|
23
20
|
s.add_development_dependency 'rake'
|
24
21
|
end
|
metadata
CHANGED
@@ -1,62 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mys3ql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
5
|
-
prerelease:
|
4
|
+
version: 1.3.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Andy Stewart
|
9
|
-
autorequire:
|
8
|
+
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2021-02-08 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
|
-
name:
|
14
|
+
name: aws-sdk-s3
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- - ~>
|
17
|
+
- - "~>"
|
20
18
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
19
|
+
version: '1'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- - ~>
|
24
|
+
- - "~>"
|
28
25
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
30
|
-
- !ruby/object:Gem::Dependency
|
31
|
-
name: fog
|
32
|
-
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
|
-
requirements:
|
35
|
-
- - ~>
|
36
|
-
- !ruby/object:Gem::Version
|
37
|
-
version: 1.19.0
|
38
|
-
type: :runtime
|
39
|
-
prerelease: false
|
40
|
-
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
|
-
requirements:
|
43
|
-
- - ~>
|
44
|
-
- !ruby/object:Gem::Version
|
45
|
-
version: 1.19.0
|
26
|
+
version: '1'
|
46
27
|
- !ruby/object:Gem::Dependency
|
47
28
|
name: rake
|
48
29
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
30
|
requirements:
|
51
|
-
- -
|
31
|
+
- - ">="
|
52
32
|
- !ruby/object:Gem::Version
|
53
33
|
version: '0'
|
54
34
|
type: :development
|
55
35
|
prerelease: false
|
56
36
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
37
|
requirements:
|
59
|
-
- -
|
38
|
+
- - ">="
|
60
39
|
- !ruby/object:Gem::Version
|
61
40
|
version: '0'
|
62
41
|
description: Simple backup of your MySql database onto Amazon S3.
|
@@ -67,8 +46,7 @@ executables:
|
|
67
46
|
extensions: []
|
68
47
|
extra_rdoc_files: []
|
69
48
|
files:
|
70
|
-
- .gitignore
|
71
|
-
- CHANGELOG.md
|
49
|
+
- ".gitignore"
|
72
50
|
- Gemfile
|
73
51
|
- README.md
|
74
52
|
- Rakefile
|
@@ -83,32 +61,25 @@ files:
|
|
83
61
|
- mys3ql.gemspec
|
84
62
|
homepage: https://github.com/airblade/mys3ql
|
85
63
|
licenses: []
|
86
|
-
|
64
|
+
metadata: {}
|
65
|
+
post_install_message:
|
87
66
|
rdoc_options: []
|
88
67
|
require_paths:
|
89
68
|
- lib
|
90
69
|
required_ruby_version: !ruby/object:Gem::Requirement
|
91
|
-
none: false
|
92
70
|
requirements:
|
93
|
-
- -
|
71
|
+
- - ">="
|
94
72
|
- !ruby/object:Gem::Version
|
95
73
|
version: '0'
|
96
|
-
segments:
|
97
|
-
- 0
|
98
|
-
hash: 271295663072223621
|
99
74
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
-
none: false
|
101
75
|
requirements:
|
102
|
-
- -
|
76
|
+
- - ">="
|
103
77
|
- !ruby/object:Gem::Version
|
104
78
|
version: '0'
|
105
|
-
segments:
|
106
|
-
- 0
|
107
|
-
hash: 271295663072223621
|
108
79
|
requirements: []
|
109
|
-
rubyforge_project:
|
110
|
-
rubygems_version:
|
111
|
-
signing_key:
|
112
|
-
specification_version:
|
80
|
+
rubyforge_project:
|
81
|
+
rubygems_version: 2.5.2.3
|
82
|
+
signing_key:
|
83
|
+
specification_version: 4
|
113
84
|
summary: Simple backup of your MySql database onto Amazon S3.
|
114
85
|
test_files: []
|
data/CHANGELOG.md
DELETED