backups-cli 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +2 -0
- data/.travis.yml +5 -0
- data/Gemfile +3 -0
- data/LICENSE +7 -0
- data/README.md +256 -0
- data/Rakefile +6 -0
- data/backups-cli.gemspec +25 -0
- data/bin/backups +47 -0
- data/lib/backups.rb +19 -0
- data/lib/backups/adapter/mysql.rb +202 -0
- data/lib/backups/base.rb +70 -0
- data/lib/backups/cli.rb +51 -0
- data/lib/backups/crontab.rb +200 -0
- data/lib/backups/driver/mysql.rb +35 -0
- data/lib/backups/events.rb +40 -0
- data/lib/backups/ext/fixnum.rb +21 -0
- data/lib/backups/ext/hash.rb +8 -0
- data/lib/backups/ext/nil_class.rb +7 -0
- data/lib/backups/ext/ordered_hash.rb +11 -0
- data/lib/backups/ext/string.rb +20 -0
- data/lib/backups/listener.rb +19 -0
- data/lib/backups/listeners/notify/datadog.rb +44 -0
- data/lib/backups/listeners/notify/slack.rb +121 -0
- data/lib/backups/loader.rb +55 -0
- data/lib/backups/logger.rb +8 -0
- data/lib/backups/runner.rb +137 -0
- data/lib/backups/stats/mysql.rb +80 -0
- data/lib/backups/system.rb +54 -0
- data/lib/backups/verify/mysql.rb +180 -0
- data/lib/backups/version.rb +4 -0
- metadata +173 -0
@@ -0,0 +1,20 @@
|
|
1
|
+
unless String.method_defined?(:to_bool)
|
2
|
+
class String
|
3
|
+
def to_bool
|
4
|
+
return true if self == true || self =~ (/^(true|t|yes|y|1)$/i)
|
5
|
+
return false if self == false || self.blank? || self =~ (/^(false|f|no|n|0)$/i)
|
6
|
+
raise ArgumentError.new("invalid value for Boolean: \"#{self}\"")
|
7
|
+
end
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
unless String.method_defined?(:squish!)
|
12
|
+
class String
|
13
|
+
def squish!
|
14
|
+
gsub!(/\A[[:space:]]+/, '')
|
15
|
+
gsub!(/[[:space:]]+\z/, '')
|
16
|
+
gsub!(/[[:space:]]+/, ' ')
|
17
|
+
self
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Backups
|
2
|
+
class Listener
|
3
|
+
|
4
|
+
def self.listen *names, &block
|
5
|
+
names.each do |name|
|
6
|
+
if block_given?
|
7
|
+
Events.on name do |params|
|
8
|
+
block.call(params)
|
9
|
+
end
|
10
|
+
else
|
11
|
+
Events.on name do |params|
|
12
|
+
notify params
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require "dogapi"
|
2
|
+
|
3
|
+
module Backups
|
4
|
+
module Listeners
|
5
|
+
module Notify
|
6
|
+
class Datadog < Listener
|
7
|
+
|
8
|
+
def initialize config
|
9
|
+
@config = config
|
10
|
+
api_key = @config.fetch("api_key")
|
11
|
+
app_key = @config.fetch("app_key")
|
12
|
+
|
13
|
+
@dog = Dogapi::Client.new(api_key, app_key)
|
14
|
+
|
15
|
+
$LOGGER.debug "Datadog listener started"
|
16
|
+
Events::on :start, :done, :error do |params|
|
17
|
+
_notify params
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
private
|
22
|
+
def _notify params
|
23
|
+
metric = @config.fetch("metric", "monitoring.backups")
|
24
|
+
tags = _flatten(params.fetch(:config, {}).fetch("tags", {}))
|
25
|
+
tags << "type:#{params[:event]}"
|
26
|
+
tags << "job:#{params[:job]}"
|
27
|
+
|
28
|
+
status, response = @dog.emit_point(metric, 1, tags: tags)
|
29
|
+
abort "Failed to send metric to Datadog." unless status == "202"
|
30
|
+
end
|
31
|
+
|
32
|
+
def _flatten tags
|
33
|
+
values = []
|
34
|
+
tags.each do |k, v|
|
35
|
+
next if v == nil
|
36
|
+
values << "#{k}:#{v}"
|
37
|
+
end
|
38
|
+
values
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require "slack-notifier"
|
2
|
+
|
3
|
+
module Backups
|
4
|
+
module Listeners
|
5
|
+
module Notify
|
6
|
+
class Slack < Listener
|
7
|
+
|
8
|
+
def initialize config
|
9
|
+
@config = config
|
10
|
+
|
11
|
+
webhook = config.fetch("webhook")
|
12
|
+
channel = config.fetch("channel", "backups")
|
13
|
+
username = config.fetch("username", "backups-cli")
|
14
|
+
events = config.fetch("events", "all")
|
15
|
+
active = config.fetch("active", true)
|
16
|
+
|
17
|
+
unless active
|
18
|
+
$LOGGER.debug "Slack listener not active"
|
19
|
+
return
|
20
|
+
end
|
21
|
+
|
22
|
+
@slack = ::Slack::Notifier.new webhook, channel: channel, username: username
|
23
|
+
$LOGGER.debug "Slack listener initialized"
|
24
|
+
|
25
|
+
if events == "all" or events.include? "start"
|
26
|
+
$LOGGER.debug "Slack listening to start events"
|
27
|
+
Events::on :start do |params|
|
28
|
+
_start params
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
if events == "all" or events.include? "complete"
|
33
|
+
$LOGGER.debug "Slack listening to complete events"
|
34
|
+
Events::on :done do |params|
|
35
|
+
_done params
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
if events == "all" or events.include? "error"
|
40
|
+
$LOGGER.debug "Slack listening to error events"
|
41
|
+
Events::on :error do |params|
|
42
|
+
_error params
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
def _tags attachment, params
|
49
|
+
params.fetch(:config, {}).fetch("tags", {}).each do |k,v|
|
50
|
+
next if v == nil
|
51
|
+
attachment[:fields] << {
|
52
|
+
title: "Tag: #{k}",
|
53
|
+
value: v,
|
54
|
+
short: true,
|
55
|
+
}
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def _done params
|
60
|
+
details = params.fetch(:details)
|
61
|
+
notes = {
|
62
|
+
color: "good",
|
63
|
+
fields: [
|
64
|
+
{
|
65
|
+
title: "Size",
|
66
|
+
value: "#{details[:size].to_filesize} (#{details[:size]} bytes)",
|
67
|
+
short: true,
|
68
|
+
},
|
69
|
+
{
|
70
|
+
title: "Took",
|
71
|
+
value: "#{(details[:completed] - details[:started]).round(2)} seconds",
|
72
|
+
short: true,
|
73
|
+
},
|
74
|
+
{
|
75
|
+
title: "File",
|
76
|
+
value: "<#{details[:view]}|#{details[:url]}>",
|
77
|
+
},
|
78
|
+
],
|
79
|
+
}
|
80
|
+
|
81
|
+
_tags notes, params
|
82
|
+
_send "Backup job `#{params[:job]}` is complete.", [notes]
|
83
|
+
end
|
84
|
+
|
85
|
+
def _error params
|
86
|
+
error = params.fetch(:error, "Something went wrong but I don't know what.")
|
87
|
+
notes = {
|
88
|
+
color: "danger",
|
89
|
+
fallback: error,
|
90
|
+
fields: [
|
91
|
+
{
|
92
|
+
title: "Error",
|
93
|
+
value: error,
|
94
|
+
},
|
95
|
+
],
|
96
|
+
}
|
97
|
+
|
98
|
+
_tags notes, params
|
99
|
+
_send "Backup job `#{params[:job]}` failed.", [notes]
|
100
|
+
end
|
101
|
+
|
102
|
+
def _start params
|
103
|
+
notes = { fields: [] }
|
104
|
+
msg = "Backup job `#{params[:job]}` has started.".encode!("UTF-8", {undef: :replace})
|
105
|
+
|
106
|
+
_tags notes, params
|
107
|
+
_send msg, [notes]
|
108
|
+
end
|
109
|
+
|
110
|
+
def _send message, attachments
|
111
|
+
res = @slack.ping message, attachments: attachments
|
112
|
+
$LOGGER.debug "Event sent to Slack: #{message}"
|
113
|
+
# if res.code
|
114
|
+
# $LOGGER.debug "Slack result: #{res.code} #{res.message}"
|
115
|
+
# end
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require "yaml"
|
2
|
+
|
3
|
+
module Backups
|
4
|
+
module Loader
|
5
|
+
|
6
|
+
CONFIG_ENV = "BACKUPS_CONFIG_DIR"
|
7
|
+
CONFIG_USER = "~/.backups-cli"
|
8
|
+
CONFIG_SYSTEM = "/etc/backups-cli"
|
9
|
+
|
10
|
+
def load_configs
|
11
|
+
config_dir = find_dir()
|
12
|
+
load_files config_dir
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
def load_files dir
|
17
|
+
$GLOBAL = {
|
18
|
+
"backups" => {},
|
19
|
+
"defaults" => {},
|
20
|
+
"jobs" => {},
|
21
|
+
}
|
22
|
+
|
23
|
+
# First, we dep merge all config into a single config
|
24
|
+
Dir["#{dir}/**/*.yaml"].each do |file|
|
25
|
+
$GLOBAL = $GLOBAL.deep_merge(YAML.load_file(file))
|
26
|
+
end
|
27
|
+
|
28
|
+
# Second, we apply the defaults config to all jobs
|
29
|
+
$GLOBAL["jobs"].each do |name, config|
|
30
|
+
$GLOBAL["jobs"][name] = $GLOBAL["defaults"].deep_merge(config)
|
31
|
+
$GLOBAL["jobs"][name]["_name"] = name
|
32
|
+
end
|
33
|
+
|
34
|
+
File.write("#{dir}/merged.compiled-yaml", $GLOBAL.to_yaml)
|
35
|
+
end
|
36
|
+
|
37
|
+
def find_dir
|
38
|
+
dirs = []
|
39
|
+
dirs << ENV.fetch(CONFIG_ENV) if ENV[CONFIG_ENV]
|
40
|
+
dirs << File.expand_path(CONFIG_USER) if ENV["HOME"]
|
41
|
+
dirs << CONFIG_SYSTEM
|
42
|
+
|
43
|
+
dirs.each do |dir|
|
44
|
+
return File.realpath(dir) if File.directory? dir
|
45
|
+
end
|
46
|
+
|
47
|
+
raise RuntimeError, <<-EOS.squish!
|
48
|
+
The config directory could not be found. You need to either set
|
49
|
+
the BACKUPS_CONFIG_DIR env var to a valid directory or create either a
|
50
|
+
#{CONFIG_USER} or a #{CONFIG_SYSTEM} directory.
|
51
|
+
EOS
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,137 @@
|
|
1
|
+
module Backups
|
2
|
+
class Runner
|
3
|
+
|
4
|
+
ADAPTER_PREFIX = "Backups::Adapter::"
|
5
|
+
VERIFY_PREFIX = "Backups::Verify::"
|
6
|
+
|
7
|
+
include System
|
8
|
+
include Loader
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
load_configs
|
12
|
+
load_listeners
|
13
|
+
end
|
14
|
+
|
15
|
+
def start_all
|
16
|
+
$LOGGER.info "Starting all jobs sequentially"
|
17
|
+
$GLOBAL["jobs"].each do |name, config|
|
18
|
+
start name
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def verify_all
|
23
|
+
$LOGGER.info "Verifying all jobs sequentially"
|
24
|
+
$GLOBAL["jobs"].each do |name, config|
|
25
|
+
verify name
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def start job
|
30
|
+
$LOGGER.progname = job
|
31
|
+
$LOGGER.info "Started backup"
|
32
|
+
|
33
|
+
config = _load_config(job)
|
34
|
+
|
35
|
+
Events.fire :start, {
|
36
|
+
job: job,
|
37
|
+
config: config,
|
38
|
+
}
|
39
|
+
|
40
|
+
details = _backup(job, config)
|
41
|
+
|
42
|
+
Events.fire :done, {
|
43
|
+
type: :backup,
|
44
|
+
job: job,
|
45
|
+
config: config,
|
46
|
+
details: details,
|
47
|
+
}
|
48
|
+
|
49
|
+
$LOGGER.info "Completed backup"
|
50
|
+
end
|
51
|
+
|
52
|
+
def verify job
|
53
|
+
$LOGGER.progname = job
|
54
|
+
$LOGGER.info "Started verification"
|
55
|
+
|
56
|
+
config = _load_config(job)
|
57
|
+
|
58
|
+
Events.fire :start, {
|
59
|
+
job: job,
|
60
|
+
config: config,
|
61
|
+
}
|
62
|
+
|
63
|
+
details = _verify(job, config)
|
64
|
+
|
65
|
+
Events.fire :done, {
|
66
|
+
type: :verify,
|
67
|
+
job: job,
|
68
|
+
config: config,
|
69
|
+
details: details,
|
70
|
+
}
|
71
|
+
|
72
|
+
$LOGGER.info "Completed verification"
|
73
|
+
end
|
74
|
+
|
75
|
+
def show job = nil
|
76
|
+
return _load_config(job) if job
|
77
|
+
$GLOBAL
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
def _load_config job
|
82
|
+
config = $GLOBAL["jobs"][job]
|
83
|
+
raise "Job #{job} is not defined." unless config
|
84
|
+
raise "Job #{job} is not enabled." unless config.fetch("enabled", true)
|
85
|
+
raise "Job #{job} has no type." unless config["type"]
|
86
|
+
|
87
|
+
config
|
88
|
+
end
|
89
|
+
|
90
|
+
def _backup job, config
|
91
|
+
type = config["type"].capitalize
|
92
|
+
klass = class_for_name("#{ADAPTER_PREFIX}#{type}")
|
93
|
+
raise RuntimeError, "Could not load the #{type} adapter." unless klass
|
94
|
+
adapter = klass.new(config)
|
95
|
+
details = adapter.run()
|
96
|
+
|
97
|
+
### Test the verify step immediately after
|
98
|
+
# details["verify"] = _verify(job, config)
|
99
|
+
|
100
|
+
details
|
101
|
+
end
|
102
|
+
|
103
|
+
def _verify job, config
|
104
|
+
type = config["type"].capitalize
|
105
|
+
klass = class_for_name("#{VERIFY_PREFIX}#{type}")
|
106
|
+
raise RuntimeError, "Could not load the #{type} verify." unless klass
|
107
|
+
adapter = klass.new(config)
|
108
|
+
adapter.verify()
|
109
|
+
end
|
110
|
+
|
111
|
+
def class_for_name name
|
112
|
+
name.split("::").inject(Object) { |o, c| o.const_get c }
|
113
|
+
end
|
114
|
+
|
115
|
+
def load_listeners
|
116
|
+
Dir["#{File.dirname(__FILE__)}/listeners/**/*.rb"].each do |file|
|
117
|
+
require file
|
118
|
+
|
119
|
+
name = file.gsub("#{File.dirname(__FILE__)}/listeners/", "")
|
120
|
+
name = name[0..-4]
|
121
|
+
full = ""
|
122
|
+
name.split("/").each do |v|
|
123
|
+
full += "::#{v.capitalize}"
|
124
|
+
end
|
125
|
+
|
126
|
+
listeners = $GLOBAL["backups"].fetch("listeners", {})
|
127
|
+
# search = name.gsub("/", ".")
|
128
|
+
search = File.basename(name)
|
129
|
+
config = listeners.fetch(search, {})
|
130
|
+
|
131
|
+
klass = class_for_name("Backups::Listeners#{full}")
|
132
|
+
listn = klass.new(config)
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module Backups
|
2
|
+
module Stats
|
3
|
+
module Mysql
|
4
|
+
|
5
|
+
include ::Backups::Driver::Mysql
|
6
|
+
|
7
|
+
def get_server_tables(databases = nil)
|
8
|
+
query = <<-EOS
|
9
|
+
SELECT
|
10
|
+
TABLE_SCHEMA `database`,
|
11
|
+
TABLE_NAME `table`
|
12
|
+
FROM
|
13
|
+
INFORMATION_SCHEMA.TABLES
|
14
|
+
WHERE
|
15
|
+
TABLE_SCHEMA NOT IN ('#{get_excluded_schemas.join("', '")}')
|
16
|
+
EOS
|
17
|
+
|
18
|
+
query += "AND TABLE_SCHEMA IN ('#{databases.join("', '")}')" if databases
|
19
|
+
query += "ORDER BY TABLE_SCHEMA ASC"
|
20
|
+
|
21
|
+
get_results(query)
|
22
|
+
end
|
23
|
+
|
24
|
+
def get_database_tables(database)
|
25
|
+
query = <<-EOS
|
26
|
+
SELECT
|
27
|
+
TABLE_NAME `table`
|
28
|
+
FROM
|
29
|
+
INFORMATION_SCHEMA.TABLES
|
30
|
+
WHERE
|
31
|
+
TABLE_SCHEMA = '#{database}'
|
32
|
+
EOS
|
33
|
+
|
34
|
+
get_results(query)
|
35
|
+
end
|
36
|
+
|
37
|
+
def get_database_stats(database)
|
38
|
+
tables = get_database_tables(database)
|
39
|
+
stats = {}
|
40
|
+
tables.each do |item|
|
41
|
+
table = item['table']
|
42
|
+
|
43
|
+
stats[table] = get_table_stats(database, table)
|
44
|
+
$LOGGER.debug "Writing stats for table #{database}.#{table}: #{stats[table]}"
|
45
|
+
end
|
46
|
+
|
47
|
+
return stats
|
48
|
+
end
|
49
|
+
|
50
|
+
def get_excluded_schemas
|
51
|
+
["information_schema", "performance_schema", "mysql"]
|
52
|
+
end
|
53
|
+
|
54
|
+
def get_database_names
|
55
|
+
query = <<-EOS
|
56
|
+
SELECT
|
57
|
+
SCHEMA_NAME `database`
|
58
|
+
FROM
|
59
|
+
INFORMATION_SCHEMA.SCHEMATA
|
60
|
+
WHERE
|
61
|
+
SCHEMA_NAME NOT IN ('#{get_excluded_schemas.join("', '")}')
|
62
|
+
EOS
|
63
|
+
|
64
|
+
get_results(query)
|
65
|
+
end
|
66
|
+
|
67
|
+
def get_table_stats(database, table)
|
68
|
+
query = <<-EOS
|
69
|
+
SELECT
|
70
|
+
COUNT(*) `rows`
|
71
|
+
FROM
|
72
|
+
`#{database}`.`#{table}`
|
73
|
+
EOS
|
74
|
+
|
75
|
+
get_result(query)
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|