muck 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +19 -0
- data/MIT-LICENCE +20 -0
- data/README.md +122 -0
- data/bin/muck +32 -0
- data/lib/muck/archive.rb +70 -0
- data/lib/muck/backup.rb +101 -0
- data/lib/muck/config.rb +43 -0
- data/lib/muck/config_dsl/database_dsl.rb +27 -0
- data/lib/muck/config_dsl/retention_dsl.rb +27 -0
- data/lib/muck/config_dsl/root_dsl.rb +25 -0
- data/lib/muck/config_dsl/server_dsl.rb +47 -0
- data/lib/muck/config_dsl/ssh_dsl.rb +23 -0
- data/lib/muck/config_dsl/storage_dsl.rb +19 -0
- data/lib/muck/database.rb +77 -0
- data/lib/muck/error.rb +4 -0
- data/lib/muck/logging.rb +20 -0
- data/lib/muck/server.rb +68 -0
- data/lib/muck/utils.rb +25 -0
- data/lib/muck/version.rb +3 -0
- data/muck.gemspec +18 -0
- metadata +87 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 03519690f84db0eeed8308449dfd2bcfb534578a
|
4
|
+
data.tar.gz: ccaabeea5580e22e8571005ed3ec0d3c6d016d26
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9dc93af8f0d95b9c2f58c0078fd672f3a5597c33cc3effdbd593ca48da97f8c32ac02176ec0a53430a83af6712996856d6089ee8f12c73a95566efeb42b3626e
|
7
|
+
data.tar.gz: 3fab96b321b514c56949f9d3d10f47db9b24bacba46df73028f01fa3ccb719726c5e9981d32af8a6c2746ee90aeb510cff7654232a97de64c2d946192d4091fc
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
data/MIT-LICENCE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright 2016 Adam Cooke.
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,122 @@
|
|
1
|
+
# Muck
|
2
|
+
|
3
|
+
Muck is a tool which will backup & store MySQL dump files from remote hosts. Through a simple configuration file, you can add hosts & databaes which you wish to be backed up and Muck will connect to those hosts over SSH, grab a dump file using `mysqldump`, gzip it and store it away on its own server.
|
4
|
+
|
5
|
+
* Connect to any number of servers and backup any number of databases on each server.
|
6
|
+
* Archive backups to ensure you retain historical backups.
|
7
|
+
* Tidies up after itself.
|
8
|
+
* Secure because we connect over SSH before connecting to the database.
|
9
|
+
* Runs as a service or in a cron.
|
10
|
+
|
11
|
+
## Requirements
|
12
|
+
|
13
|
+
* Ruby 2.3 or higher
|
14
|
+
|
15
|
+
## Installation
|
16
|
+
|
17
|
+
```
|
18
|
+
sudo gem install muck
|
19
|
+
```
|
20
|
+
|
21
|
+
We recommend taht you create a user which will run your Muck services.
|
22
|
+
|
23
|
+
```
|
24
|
+
sudo useradd -r -m -d /opt/muck muck
|
25
|
+
```
|
26
|
+
|
27
|
+
You'll need to make directories for configuration and storage.
|
28
|
+
|
29
|
+
```
|
30
|
+
sudo -u muck mkdir /opt/muck/config
|
31
|
+
sudo -u muck mkdir /opt/muck/storage
|
32
|
+
```
|
33
|
+
|
34
|
+
Finally, you'll need to generate an SSH key pair which will be used for authenticating your requests to the servers you wish to backup. Password authentication is not supported in Muck.
|
35
|
+
|
36
|
+
```
|
37
|
+
sudo -u muck ssh-keygen -f /opt/muck/ssh-key
|
38
|
+
# Follow the instructions to generate a keypair. Do not add a passphrase.
|
39
|
+
```
|
40
|
+
|
41
|
+
## Configuration
|
42
|
+
|
43
|
+
We recommend storing your muck configuration in `/opt/muck/config`. You should add a single file for each server you wish to backup. This is a full example file which includes all configuration options which are available. Sensible defaults are set too so most options can be skipped. The values in the example below are the current defaults.
|
44
|
+
|
45
|
+
```ruby
|
46
|
+
server do
|
47
|
+
# The hostname of the server you wish to backup. Used to connect with SSH and
|
48
|
+
# the name of the directory used for storing the backups.
|
49
|
+
hostname "myserver.example.com"
|
50
|
+
|
51
|
+
# How often you wish to take a backup (in minutes)
|
52
|
+
frequency 60
|
53
|
+
|
54
|
+
ssh do
|
55
|
+
# The user that should connect to the server with SSH
|
56
|
+
username 'root'
|
57
|
+
# The SSH port
|
58
|
+
port 22
|
59
|
+
# The path to the SSH key that you will authenticate with
|
60
|
+
key "/opt/muck/ssh-key"
|
61
|
+
end
|
62
|
+
|
63
|
+
storage do
|
64
|
+
# Specifies the directory that backups will be stored for this server. You
|
65
|
+
# can use :hostname to insert the name of the hostname automatically and
|
66
|
+
# :database to insert the database name.
|
67
|
+
path "/opt/muck/data/:hostname/:database"
|
68
|
+
# The number of "master" bacups which should be kept before being archived.
|
69
|
+
keep 50
|
70
|
+
end
|
71
|
+
|
72
|
+
retention do
|
73
|
+
# How many hourly backups do you wish to keep?
|
74
|
+
hourly 24
|
75
|
+
# How many daily backups do you wish to keep?
|
76
|
+
daily 7
|
77
|
+
# How many monthly backups do you wish to keep?
|
78
|
+
monthly 12
|
79
|
+
# How many yearly backups do you wish to keep
|
80
|
+
yearly 8
|
81
|
+
end
|
82
|
+
|
83
|
+
database do
|
84
|
+
# The name of the database
|
85
|
+
name "example"
|
86
|
+
# The hostname (as accessed from the server) to connect to
|
87
|
+
hostname "127.0.0.1"
|
88
|
+
# The username to authenticate to MySQL with
|
89
|
+
username "root"
|
90
|
+
# The password to authenticate to MySQL with
|
91
|
+
password nil
|
92
|
+
end
|
93
|
+
|
94
|
+
# The database block above can be repeated within the context of the server
|
95
|
+
# to backup multiple databases from the same server.
|
96
|
+
|
97
|
+
end
|
98
|
+
```
|
99
|
+
|
100
|
+
## Running Backups
|
101
|
+
|
102
|
+
The `muck` command line tool can be used in two ways.
|
103
|
+
|
104
|
+
* `muck start` - this will run constantly (and can be backgrounded to turned into a service as appropriate). It will respect the `frequency` option specified for a server and back all servers up whenever they are due for a backup.
|
105
|
+
* `muck run` - this will take a backup from all servers & database and exit when complete.
|
106
|
+
|
107
|
+
Both ways will send all log output to STDOUT.
|
108
|
+
|
109
|
+
## Data
|
110
|
+
|
111
|
+
The data directory will populate itself as follows:
|
112
|
+
|
113
|
+
* `data/master` - this stores each raw backup as it is downloaded (gzipped)
|
114
|
+
* `data/hourly` - this stores the hourly backups
|
115
|
+
* `data/daily` - this stores the daily backups
|
116
|
+
* `data/monthly` - this stores the monthly backups
|
117
|
+
* `data/yearly` - this stores the yearly backups
|
118
|
+
* `data/manifest.yml` - this stores a list of each master backup with a timestamp and a size
|
119
|
+
|
120
|
+
## Changing the defaults
|
121
|
+
|
122
|
+
If you wish to change the global defaults, you can create a file in your config directory which includes a `defaults` block. This is the same as the `server` block shown above however the word `server` on the first line should be replaced with `defaults`. Any values you add to the defaults block will be used instead of the system defaults.
|
data/bin/muck
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.push File.expand_path("../../lib", __FILE__)
|
4
|
+
|
5
|
+
config_file_path = ENV['MUCK_CONFIG_PATH'] || File.expand_path('/opt/muck/config')
|
6
|
+
|
7
|
+
require 'muck/config'
|
8
|
+
require 'muck/logging'
|
9
|
+
|
10
|
+
config = Muck::Config.new(config_file_path)
|
11
|
+
|
12
|
+
begin
|
13
|
+
case ARGV[0]
|
14
|
+
when 'run'
|
15
|
+
config.run(:force => true)
|
16
|
+
when 'start'
|
17
|
+
$running = false
|
18
|
+
Signal.trap("INT") { $exit = true; $running ? nil : exit(0) }
|
19
|
+
Signal.trap("TERM") { $exit = true; $running ? nil : exit(0) }
|
20
|
+
Muck.logger.info "\e[32mStarted Muck\e[0m"
|
21
|
+
loop do
|
22
|
+
$running = true
|
23
|
+
config.run
|
24
|
+
$running = false
|
25
|
+
$exit ? exit(0) : sleep(30)
|
26
|
+
end
|
27
|
+
else
|
28
|
+
puts "usage: #{$0} [command]"
|
29
|
+
end
|
30
|
+
rescue Muck::Error => e
|
31
|
+
puts "\e[31mError: #{e.message}\e[0m"
|
32
|
+
end
|
data/lib/muck/archive.rb
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'muck/logging'
|
2
|
+
require 'muck/utils'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module Muck
|
6
|
+
class Archive
|
7
|
+
|
8
|
+
include Muck::Logging
|
9
|
+
include Muck::Utils
|
10
|
+
|
11
|
+
MAPPING = {
|
12
|
+
:hourly => 'YYYY-mm-dd-HH',
|
13
|
+
:daily => "YYYY-mm-dd",
|
14
|
+
:monthly => 'YYYY-mm',
|
15
|
+
:yearly => 'YYYY'
|
16
|
+
}
|
17
|
+
|
18
|
+
def initialize(database, name, maximum)
|
19
|
+
@database = database
|
20
|
+
@name = name
|
21
|
+
@maximum = maximum
|
22
|
+
end
|
23
|
+
|
24
|
+
def export_path
|
25
|
+
File.join(@database.export_path, @name.to_s)
|
26
|
+
end
|
27
|
+
|
28
|
+
def run
|
29
|
+
if last_backup = @database.manifest[:backups].last
|
30
|
+
create_archive(last_backup)
|
31
|
+
tidy
|
32
|
+
else
|
33
|
+
log.info "There is no backup to archive"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def create_archive(backup)
|
38
|
+
logger.info "Archiving #{blue @name} backup for #{blue @database.name} on #{blue @database.server.hostname}"
|
39
|
+
logger.info "Using backup from #{blue backup[:path]}"
|
40
|
+
filename = filename_for(backup[:path])
|
41
|
+
archive_path = File.join(export_path, filename)
|
42
|
+
FileUtils.mkdir_p(File.dirname(archive_path))
|
43
|
+
if system("ln -f #{backup[:path]} #{archive_path}")
|
44
|
+
logger.info "Successfully stored archive at #{green archive_path}"
|
45
|
+
else
|
46
|
+
logger.error red("Couldn't store archive at #{archive_path}")
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def tidy
|
51
|
+
files = Dir[File.join(export_path, '*')].sort.reverse.drop(@maximum)
|
52
|
+
files.each do |file|
|
53
|
+
if system("rm #{file}")
|
54
|
+
logger.info "Tidied #{green file}"
|
55
|
+
else
|
56
|
+
logger.error red("Couldn't remove un-retained file at #{file}")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def filename_for(path)
|
64
|
+
name, extensions = path.split('/').last.split('.', 2)
|
65
|
+
size = MAPPING[@name.to_sym].size
|
66
|
+
name[0,size] + ".#{extensions}"
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
data/lib/muck/backup.rb
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
require 'muck/logging'
|
2
|
+
require 'muck/utils'
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
module Muck
|
6
|
+
class Backup
|
7
|
+
|
8
|
+
include Muck::Logging
|
9
|
+
include Muck::Utils
|
10
|
+
|
11
|
+
def initialize(database)
|
12
|
+
@database = database
|
13
|
+
@time = Time.now
|
14
|
+
end
|
15
|
+
|
16
|
+
def export_path
|
17
|
+
@export_path ||= File.join(@database.export_path, "master", @time.strftime("%Y-%m-%d-%H-%M-%S.sql"))
|
18
|
+
end
|
19
|
+
|
20
|
+
def run
|
21
|
+
logger.info "Backing up #{blue @database.name} from #{blue @database.server.hostname}"
|
22
|
+
take_backup
|
23
|
+
compress
|
24
|
+
store_in_manifest
|
25
|
+
tidy_masters
|
26
|
+
end
|
27
|
+
|
28
|
+
def take_backup
|
29
|
+
logger.info "Connecting to #{blue @database.server.ssh_username}@#{blue @database.server.hostname}:#{blue @database.server.ssh_port}"
|
30
|
+
FileUtils.mkdir_p(File.dirname(self.export_path))
|
31
|
+
file = File.open(export_path, 'w')
|
32
|
+
ssh_session = @database.server.create_ssh_session
|
33
|
+
channel = ssh_session.open_channel do |channel|
|
34
|
+
logger.debug "Running: #{@database.dump_command}"
|
35
|
+
channel.exec(@database.dump_command) do |channel, success|
|
36
|
+
raise Error, "Could not execute dump command" unless success
|
37
|
+
channel.on_data do |c, data|
|
38
|
+
file.write(data)
|
39
|
+
end
|
40
|
+
|
41
|
+
channel.on_extended_data do |c, _, data|
|
42
|
+
logger.debug red(data.gsub(/[\r\n]/, ''))
|
43
|
+
end
|
44
|
+
|
45
|
+
channel.on_request("exit-status") do |_, data|
|
46
|
+
exit_code = data.read_long
|
47
|
+
if exit_code != 0
|
48
|
+
logger.debug "Exit status was #{exit_code}"
|
49
|
+
raise Error, "mysqldump returned an error when executing."
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
channel.wait
|
55
|
+
ssh_session.close
|
56
|
+
file.close
|
57
|
+
logger.info "Successfully backed up to #{green export_path}"
|
58
|
+
end
|
59
|
+
|
60
|
+
def store_in_manifest
|
61
|
+
if File.exist?(export_path)
|
62
|
+
details = {:timestamp => @time.to_i, :path => export_path, :size => File.size(export_path)}
|
63
|
+
@database.manifest[:backups] << details
|
64
|
+
@database.save_manifest
|
65
|
+
else
|
66
|
+
raise Error, "Couldn't store backup in manifest because it doesn't exist at #{export_path}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def compress
|
71
|
+
if File.exist?(export_path)
|
72
|
+
if system("gzip #{export_path}")
|
73
|
+
@export_path = @export_path + ".gz"
|
74
|
+
logger.info "Compressed #{blue export_path} with gzip"
|
75
|
+
else
|
76
|
+
logger.warn "Couldn't compress #{export_path} with gzip"
|
77
|
+
end
|
78
|
+
else
|
79
|
+
raise Error, "Couldn't compress backup because it doesn't exist at #{export_path}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def tidy_masters
|
84
|
+
files = Dir[File.join(@database.export_path, 'master', '*')].sort.reverse.drop(@database.server.masters_to_keep)
|
85
|
+
unless files.empty?
|
86
|
+
logger.info "Tidying master backup files. Keeping #{@database.server.masters_to_keep} back."
|
87
|
+
files.each do |file|
|
88
|
+
if system("rm #{file}")
|
89
|
+
@database.manifest[:backups].delete_if { |b| b[:path] == file }
|
90
|
+
logger.info "-> Removed #{green file}"
|
91
|
+
else
|
92
|
+
logger.error red("-> Couldn't remove unwanted master file at #{file}")
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
ensure
|
97
|
+
@database.save_manifest
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
101
|
+
end
|
data/lib/muck/config.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'muck/error'
|
2
|
+
require 'muck/config_dsl/root_dsl'
|
3
|
+
|
4
|
+
module Muck
|
5
|
+
class Config
|
6
|
+
|
7
|
+
def initialize(directory)
|
8
|
+
@directory = directory
|
9
|
+
@defaults = {}
|
10
|
+
@servers = []
|
11
|
+
parse
|
12
|
+
end
|
13
|
+
|
14
|
+
attr_reader :defaults
|
15
|
+
attr_reader :servers
|
16
|
+
|
17
|
+
def run(options = {})
|
18
|
+
servers.each do |server|
|
19
|
+
server.databases.each do |database|
|
20
|
+
if database.backup_now? || options[:force]
|
21
|
+
database.backup
|
22
|
+
database.archive_all
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def parse
|
31
|
+
unless File.directory?(@directory)
|
32
|
+
raise Muck::Error, "#{@directory} is not a directory"
|
33
|
+
end
|
34
|
+
|
35
|
+
root_dsl = ConfigDSL::RootDSL.new(self)
|
36
|
+
files = Dir[File.join(@directory, "**", "*.rb")]
|
37
|
+
files.each do |file|
|
38
|
+
root_dsl.instance_eval(File.read(file), file)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Muck
|
2
|
+
module ConfigDSL
|
3
|
+
class DatabaseDSL
|
4
|
+
|
5
|
+
def initialize(hash)
|
6
|
+
@hash = hash
|
7
|
+
end
|
8
|
+
|
9
|
+
def name(name)
|
10
|
+
@hash[:name] = name
|
11
|
+
end
|
12
|
+
|
13
|
+
def hostname(hostname)
|
14
|
+
@hash[:hostname] = hostname
|
15
|
+
end
|
16
|
+
|
17
|
+
def username(username)
|
18
|
+
@hash[:username] = username
|
19
|
+
end
|
20
|
+
|
21
|
+
def password(password)
|
22
|
+
@hash[:password] = password
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Muck
|
2
|
+
module ConfigDSL
|
3
|
+
class RetentionDSL
|
4
|
+
|
5
|
+
def initialize(hash)
|
6
|
+
@hash = hash
|
7
|
+
end
|
8
|
+
|
9
|
+
def hourly(hourly)
|
10
|
+
@hash[:hourly] = hourly
|
11
|
+
end
|
12
|
+
|
13
|
+
def daily(daily)
|
14
|
+
@hash[:daily] = daily
|
15
|
+
end
|
16
|
+
|
17
|
+
def monthly(monthly)
|
18
|
+
@hash[:monthly] = monthly
|
19
|
+
end
|
20
|
+
|
21
|
+
def yearly(yearly)
|
22
|
+
@hash[:yearly] = yearly
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'muck/config_dsl/server_dsl'
|
2
|
+
require 'muck/server'
|
3
|
+
module Muck
|
4
|
+
module ConfigDSL
|
5
|
+
class RootDSL
|
6
|
+
|
7
|
+
def initialize(config)
|
8
|
+
@config = config
|
9
|
+
end
|
10
|
+
|
11
|
+
def server(&block)
|
12
|
+
hash = Hash.new
|
13
|
+
dsl = ServerDSL.new(hash)
|
14
|
+
dsl.instance_eval(&block)
|
15
|
+
@config.servers << Server.new(@config, hash)
|
16
|
+
end
|
17
|
+
|
18
|
+
def defaults(&block)
|
19
|
+
dsl = ServerDSL.new(@config.defaults)
|
20
|
+
dsl.instance_eval(&block)
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'muck/config_dsl/ssh_dsl'
|
2
|
+
require 'muck/config_dsl/storage_dsl'
|
3
|
+
require 'muck/config_dsl/retention_dsl'
|
4
|
+
require 'muck/config_dsl/database_dsl'
|
5
|
+
|
6
|
+
module Muck
|
7
|
+
module ConfigDSL
|
8
|
+
class ServerDSL
|
9
|
+
|
10
|
+
def initialize(hash)
|
11
|
+
@hash = hash
|
12
|
+
end
|
13
|
+
|
14
|
+
def hostname(hostname)
|
15
|
+
@hash[:hostname] = hostname
|
16
|
+
end
|
17
|
+
|
18
|
+
def frequency(frequency)
|
19
|
+
@hash[:frequency] = frequency
|
20
|
+
end
|
21
|
+
|
22
|
+
def ssh(&block)
|
23
|
+
dsl = SSHDSL.new(@hash[:ssh] = Hash.new)
|
24
|
+
dsl.instance_eval(&block)
|
25
|
+
end
|
26
|
+
|
27
|
+
def storage(&block)
|
28
|
+
dsl = StorageDSL.new(@hash[:storage] = Hash.new)
|
29
|
+
dsl.instance_eval(&block)
|
30
|
+
end
|
31
|
+
|
32
|
+
def retention(&block)
|
33
|
+
dsl = RetentionDSL.new(@hash[:retention] = Hash.new)
|
34
|
+
dsl.instance_eval(&block)
|
35
|
+
end
|
36
|
+
|
37
|
+
def database(&block)
|
38
|
+
hash = {}
|
39
|
+
dsl = DatabaseDSL.new(hash)
|
40
|
+
dsl.instance_eval(&block)
|
41
|
+
@hash[:databases] ||= []
|
42
|
+
@hash[:databases] << hash
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Muck
|
2
|
+
module ConfigDSL
|
3
|
+
class SSHDSL
|
4
|
+
|
5
|
+
def initialize(hash)
|
6
|
+
@hash = hash
|
7
|
+
end
|
8
|
+
|
9
|
+
def port(port)
|
10
|
+
@hash[:port] = port
|
11
|
+
end
|
12
|
+
|
13
|
+
def key(key)
|
14
|
+
@hash[:key] = key
|
15
|
+
end
|
16
|
+
|
17
|
+
def username(username)
|
18
|
+
@hash[:username] = username
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'muck/archive'
|
2
|
+
require 'muck/backup'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module Muck
|
6
|
+
class Database
|
7
|
+
|
8
|
+
def initialize(server, properties)
|
9
|
+
@server = server
|
10
|
+
@properties = properties
|
11
|
+
end
|
12
|
+
|
13
|
+
def name
|
14
|
+
@properties[:name]
|
15
|
+
end
|
16
|
+
|
17
|
+
def hostname
|
18
|
+
@properties[:hostname]
|
19
|
+
end
|
20
|
+
|
21
|
+
def username
|
22
|
+
@properties[:username]
|
23
|
+
end
|
24
|
+
|
25
|
+
def password
|
26
|
+
@properties[:password]
|
27
|
+
end
|
28
|
+
|
29
|
+
def server
|
30
|
+
@server
|
31
|
+
end
|
32
|
+
|
33
|
+
def export_path
|
34
|
+
@export_path ||= server.export_path.gsub(':database', self.name)
|
35
|
+
end
|
36
|
+
|
37
|
+
def archive_all
|
38
|
+
@server.retention.each do |name, maximum|
|
39
|
+
Muck::Archive.new(self, name, maximum).run
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def backup
|
44
|
+
Muck::Backup.new(self).run
|
45
|
+
end
|
46
|
+
|
47
|
+
def manifest_path
|
48
|
+
File.join(export_path, 'manifest.yml')
|
49
|
+
end
|
50
|
+
|
51
|
+
def manifest
|
52
|
+
@manifest ||= File.exist?(manifest_path) ? YAML.load_file(manifest_path) : {:backups => []}
|
53
|
+
end
|
54
|
+
|
55
|
+
def save_manifest
|
56
|
+
File.open(manifest_path, 'w') { |f| f.write(manifest.to_yaml) }
|
57
|
+
end
|
58
|
+
|
59
|
+
def dump_command
|
60
|
+
password_opt = password ? "-p#{password}" : ""
|
61
|
+
"mysqldump -q --single-transaction -h #{hostname} -u #{username} #{password_opt} #{name}"
|
62
|
+
end
|
63
|
+
|
64
|
+
def last_backup_at
|
65
|
+
if last_backup = manifest[:backups].last
|
66
|
+
Time.at(last_backup[:timestamp])
|
67
|
+
else
|
68
|
+
nil
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def backup_now?
|
73
|
+
last_backup_at.nil? || last_backup_at <= Time.now - (@server.frequency * 60)
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
end
|
data/lib/muck/error.rb
ADDED
data/lib/muck/logging.rb
ADDED
data/lib/muck/server.rb
ADDED
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'muck/database'
|
2
|
+
require 'net/ssh'
|
3
|
+
|
4
|
+
module Muck
|
5
|
+
class Server
|
6
|
+
|
7
|
+
DEFAULT_RENTENTION = {:hourly => 24, :daily => 7, :monthly => 12, :yearly => 8}
|
8
|
+
DEFAULT_SSH_PROPERTIES = {:username => "root", :port => 22, :key => "/opt/muck/ssh-key"}
|
9
|
+
DEFAULT_DATABASE_PROPERTIES = {:hostname => '127.0.0.1', :username => 'root', :name => 'example', :password => nil}
|
10
|
+
|
11
|
+
def initialize(config, server_hash = {})
|
12
|
+
@config = config
|
13
|
+
@server_hash = server_hash
|
14
|
+
end
|
15
|
+
|
16
|
+
def hostname
|
17
|
+
@server_hash[:hostname]
|
18
|
+
end
|
19
|
+
|
20
|
+
def frequency
|
21
|
+
@server_hash[:frequency] || @config.defaults[:frequency] || 60
|
22
|
+
end
|
23
|
+
|
24
|
+
def export_path
|
25
|
+
if path = (@server_hash.dig(:storage, :path) || @config.defaults.dig(:storage, :path))
|
26
|
+
path.gsub(":hostname", self.hostname)
|
27
|
+
else
|
28
|
+
"/opt/muck/data/#{self.hostname}/:database"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def masters_to_keep
|
33
|
+
@server_hash.dig(:storage, :keep) || @config.defaults.dig(:storage, :keep) || 50
|
34
|
+
end
|
35
|
+
|
36
|
+
def ssh_port
|
37
|
+
ssh_properties[:port]
|
38
|
+
end
|
39
|
+
|
40
|
+
def ssh_username
|
41
|
+
ssh_properties[:username]
|
42
|
+
end
|
43
|
+
|
44
|
+
def ssh_properties
|
45
|
+
DEFAULT_SSH_PROPERTIES.merge(@config.defaults[:ssh] || {}).merge(@server_hash[:ssh] || {})
|
46
|
+
end
|
47
|
+
|
48
|
+
def retention
|
49
|
+
DEFAULT_RENTENTION.merge(@config.defaults[:retention] || {}).merge(@server_hash[:retention] || {})
|
50
|
+
end
|
51
|
+
|
52
|
+
def databases
|
53
|
+
defaults = DEFAULT_DATABASE_PROPERTIES.merge(@config.defaults[:databases]&.first || {})
|
54
|
+
if @server_hash[:databases].is_a?(Array)
|
55
|
+
@server_hash[:databases].map do |database|
|
56
|
+
Database.new(self, defaults.merge(database))
|
57
|
+
end
|
58
|
+
else
|
59
|
+
[]
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def create_ssh_session
|
64
|
+
Net::SSH.start(self.hostname, self.ssh_username, :port => self.ssh_port, :keys => ssh_properties[:key] ? [ssh_properties[:key]] : nil)
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
end
|
data/lib/muck/utils.rb
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
module Muck
|
2
|
+
module Utils
|
3
|
+
|
4
|
+
def red(text)
|
5
|
+
"\e[31m#{text}\e[0m"
|
6
|
+
end
|
7
|
+
|
8
|
+
def green(text)
|
9
|
+
"\e[32m#{text}\e[0m"
|
10
|
+
end
|
11
|
+
|
12
|
+
def yellow(text)
|
13
|
+
"\e[33m#{text}\e[0m"
|
14
|
+
end
|
15
|
+
|
16
|
+
def blue(text)
|
17
|
+
"\e[34m#{text}\e[0m"
|
18
|
+
end
|
19
|
+
|
20
|
+
def pink(text)
|
21
|
+
"\e[35m#{text}\e[0m"
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
data/lib/muck/version.rb
ADDED
data/muck.gemspec
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
$:.push File.expand_path("../lib", __FILE__)
|
2
|
+
|
3
|
+
require "muck/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "muck"
|
7
|
+
s.version = Muck::VERSION
|
8
|
+
s.authors = ["Adam Cooke"]
|
9
|
+
s.email = ["adam@atechmedia.com"]
|
10
|
+
s.homepage = "http://adamcooke.io"
|
11
|
+
s.licenses = ['MIT']
|
12
|
+
s.summary = "A tool to handle the backup & storage of remote MySQL databases."
|
13
|
+
s.description = "This tool will automatically backup & store MySQL dump files from remote servers."
|
14
|
+
s.files = Dir["**/*"]
|
15
|
+
s.bindir = "bin"
|
16
|
+
s.executables << 'muck'
|
17
|
+
s.add_dependency "net-ssh", '>= 3.2', '< 4.0'
|
18
|
+
end
|
metadata
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: muck
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Adam Cooke
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-09-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: net-ssh
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '3.2'
|
20
|
+
- - <
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '4.0'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '3.2'
|
30
|
+
- - <
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '4.0'
|
33
|
+
description: This tool will automatically backup & store MySQL dump files from remote
|
34
|
+
servers.
|
35
|
+
email:
|
36
|
+
- adam@atechmedia.com
|
37
|
+
executables:
|
38
|
+
- muck
|
39
|
+
extensions: []
|
40
|
+
extra_rdoc_files: []
|
41
|
+
files:
|
42
|
+
- bin/muck
|
43
|
+
- Gemfile
|
44
|
+
- Gemfile.lock
|
45
|
+
- lib/muck/archive.rb
|
46
|
+
- lib/muck/backup.rb
|
47
|
+
- lib/muck/config.rb
|
48
|
+
- lib/muck/config_dsl/database_dsl.rb
|
49
|
+
- lib/muck/config_dsl/retention_dsl.rb
|
50
|
+
- lib/muck/config_dsl/root_dsl.rb
|
51
|
+
- lib/muck/config_dsl/server_dsl.rb
|
52
|
+
- lib/muck/config_dsl/ssh_dsl.rb
|
53
|
+
- lib/muck/config_dsl/storage_dsl.rb
|
54
|
+
- lib/muck/database.rb
|
55
|
+
- lib/muck/error.rb
|
56
|
+
- lib/muck/logging.rb
|
57
|
+
- lib/muck/server.rb
|
58
|
+
- lib/muck/utils.rb
|
59
|
+
- lib/muck/version.rb
|
60
|
+
- MIT-LICENCE
|
61
|
+
- muck.gemspec
|
62
|
+
- README.md
|
63
|
+
homepage: http://adamcooke.io
|
64
|
+
licenses:
|
65
|
+
- MIT
|
66
|
+
metadata: {}
|
67
|
+
post_install_message:
|
68
|
+
rdoc_options: []
|
69
|
+
require_paths:
|
70
|
+
- lib
|
71
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - '>='
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '0'
|
81
|
+
requirements: []
|
82
|
+
rubyforge_project:
|
83
|
+
rubygems_version: 2.0.14.1
|
84
|
+
signing_key:
|
85
|
+
specification_version: 4
|
86
|
+
summary: A tool to handle the backup & storage of remote MySQL databases.
|
87
|
+
test_files: []
|