backups-cli 1.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,20 @@
1
+ unless String.method_defined?(:to_bool)
2
+ class String
3
+ def to_bool
4
+ return true if self == true || self =~ (/^(true|t|yes|y|1)$/i)
5
+ return false if self == false || self.blank? || self =~ (/^(false|f|no|n|0)$/i)
6
+ raise ArgumentError.new("invalid value for Boolean: \"#{self}\"")
7
+ end
8
+ end
9
+ end
10
+
11
+ unless String.method_defined?(:squish!)
12
+ class String
13
+ def squish!
14
+ gsub!(/\A[[:space:]]+/, '')
15
+ gsub!(/[[:space:]]+\z/, '')
16
+ gsub!(/[[:space:]]+/, ' ')
17
+ self
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,19 @@
1
+ module Backups
2
+ class Listener
3
+
4
+ def self.listen *names, &block
5
+ names.each do |name|
6
+ if block_given?
7
+ Events.on name do |params|
8
+ block.call(params)
9
+ end
10
+ else
11
+ Events.on name do |params|
12
+ notify params
13
+ end
14
+ end
15
+ end
16
+ end
17
+
18
+ end
19
+ end
@@ -0,0 +1,44 @@
1
+ require "dogapi"
2
+
3
+ module Backups
4
+ module Listeners
5
+ module Notify
6
+ class Datadog < Listener
7
+
8
+ def initialize config
9
+ @config = config
10
+ api_key = @config.fetch("api_key")
11
+ app_key = @config.fetch("app_key")
12
+
13
+ @dog = Dogapi::Client.new(api_key, app_key)
14
+
15
+ $LOGGER.debug "Datadog listener started"
16
+ Events::on :start, :done, :error do |params|
17
+ _notify params
18
+ end
19
+ end
20
+
21
+ private
22
+ def _notify params
23
+ metric = @config.fetch("metric", "monitoring.backups")
24
+ tags = _flatten(params.fetch(:config, {}).fetch("tags", {}))
25
+ tags << "type:#{params[:event]}"
26
+ tags << "job:#{params[:job]}"
27
+
28
+ status, response = @dog.emit_point(metric, 1, tags: tags)
29
+ abort "Failed to send metric to Datadog." unless status == "202"
30
+ end
31
+
32
+ def _flatten tags
33
+ values = []
34
+ tags.each do |k, v|
35
+ next if v == nil
36
+ values << "#{k}:#{v}"
37
+ end
38
+ values
39
+ end
40
+ end
41
+
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,121 @@
1
+ require "slack-notifier"
2
+
3
+ module Backups
4
+ module Listeners
5
+ module Notify
6
+ class Slack < Listener
7
+
8
+ def initialize config
9
+ @config = config
10
+
11
+ webhook = config.fetch("webhook")
12
+ channel = config.fetch("channel", "backups")
13
+ username = config.fetch("username", "backups-cli")
14
+ events = config.fetch("events", "all")
15
+ active = config.fetch("active", true)
16
+
17
+ unless active
18
+ $LOGGER.debug "Slack listener not active"
19
+ return
20
+ end
21
+
22
+ @slack = ::Slack::Notifier.new webhook, channel: channel, username: username
23
+ $LOGGER.debug "Slack listener initialized"
24
+
25
+ if events == "all" or events.include? "start"
26
+ $LOGGER.debug "Slack listening to start events"
27
+ Events::on :start do |params|
28
+ _start params
29
+ end
30
+ end
31
+
32
+ if events == "all" or events.include? "complete"
33
+ $LOGGER.debug "Slack listening to complete events"
34
+ Events::on :done do |params|
35
+ _done params
36
+ end
37
+ end
38
+
39
+ if events == "all" or events.include? "error"
40
+ $LOGGER.debug "Slack listening to error events"
41
+ Events::on :error do |params|
42
+ _error params
43
+ end
44
+ end
45
+ end
46
+
47
+ private
48
+ def _tags attachment, params
49
+ params.fetch(:config, {}).fetch("tags", {}).each do |k,v|
50
+ next if v == nil
51
+ attachment[:fields] << {
52
+ title: "Tag: #{k}",
53
+ value: v,
54
+ short: true,
55
+ }
56
+ end
57
+ end
58
+
59
+ def _done params
60
+ details = params.fetch(:details)
61
+ notes = {
62
+ color: "good",
63
+ fields: [
64
+ {
65
+ title: "Size",
66
+ value: "#{details[:size].to_filesize} (#{details[:size]} bytes)",
67
+ short: true,
68
+ },
69
+ {
70
+ title: "Took",
71
+ value: "#{(details[:completed] - details[:started]).round(2)} seconds",
72
+ short: true,
73
+ },
74
+ {
75
+ title: "File",
76
+ value: "<#{details[:view]}|#{details[:url]}>",
77
+ },
78
+ ],
79
+ }
80
+
81
+ _tags notes, params
82
+ _send "Backup job `#{params[:job]}` is complete.", [notes]
83
+ end
84
+
85
+ def _error params
86
+ error = params.fetch(:error, "Something went wrong but I don't know what.")
87
+ notes = {
88
+ color: "danger",
89
+ fallback: error,
90
+ fields: [
91
+ {
92
+ title: "Error",
93
+ value: error,
94
+ },
95
+ ],
96
+ }
97
+
98
+ _tags notes, params
99
+ _send "Backup job `#{params[:job]}` failed.", [notes]
100
+ end
101
+
102
+ def _start params
103
+ notes = { fields: [] }
104
+ msg = "Backup job `#{params[:job]}` has started.".encode!("UTF-8", {undef: :replace})
105
+
106
+ _tags notes, params
107
+ _send msg, [notes]
108
+ end
109
+
110
+ def _send message, attachments
111
+ res = @slack.ping message, attachments: attachments
112
+ $LOGGER.debug "Event sent to Slack: #{message}"
113
+ # if res.code
114
+ # $LOGGER.debug "Slack result: #{res.code} #{res.message}"
115
+ # end
116
+ end
117
+
118
+ end
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,55 @@
1
+ require "yaml"
2
+
3
+ module Backups
4
+ module Loader
5
+
6
+ CONFIG_ENV = "BACKUPS_CONFIG_DIR"
7
+ CONFIG_USER = "~/.backups-cli"
8
+ CONFIG_SYSTEM = "/etc/backups-cli"
9
+
10
+ def load_configs
11
+ config_dir = find_dir()
12
+ load_files config_dir
13
+ end
14
+
15
+ private
16
+ def load_files dir
17
+ $GLOBAL = {
18
+ "backups" => {},
19
+ "defaults" => {},
20
+ "jobs" => {},
21
+ }
22
+
23
+ # First, we dep merge all config into a single config
24
+ Dir["#{dir}/**/*.yaml"].each do |file|
25
+ $GLOBAL = $GLOBAL.deep_merge(YAML.load_file(file))
26
+ end
27
+
28
+ # Second, we apply the defaults config to all jobs
29
+ $GLOBAL["jobs"].each do |name, config|
30
+ $GLOBAL["jobs"][name] = $GLOBAL["defaults"].deep_merge(config)
31
+ $GLOBAL["jobs"][name]["_name"] = name
32
+ end
33
+
34
+ File.write("#{dir}/merged.compiled-yaml", $GLOBAL.to_yaml)
35
+ end
36
+
37
+ def find_dir
38
+ dirs = []
39
+ dirs << ENV.fetch(CONFIG_ENV) if ENV[CONFIG_ENV]
40
+ dirs << File.expand_path(CONFIG_USER) if ENV["HOME"]
41
+ dirs << CONFIG_SYSTEM
42
+
43
+ dirs.each do |dir|
44
+ return File.realpath(dir) if File.directory? dir
45
+ end
46
+
47
+ raise RuntimeError, <<-EOS.squish!
48
+ The config directory could not be found. You need to either set
49
+ the BACKUPS_CONFIG_DIR env var to a valid directory or create either a
50
+ #{CONFIG_USER} or a #{CONFIG_SYSTEM} directory.
51
+ EOS
52
+ end
53
+
54
+ end
55
+ end
@@ -0,0 +1,8 @@
1
+ module Backups
2
+ class Logger
3
+
4
+ def initialize
5
+ end
6
+
7
+ end
8
+ end
@@ -0,0 +1,137 @@
1
+ module Backups
2
+ class Runner
3
+
4
+ ADAPTER_PREFIX = "Backups::Adapter::"
5
+ VERIFY_PREFIX = "Backups::Verify::"
6
+
7
+ include System
8
+ include Loader
9
+
10
+ def initialize
11
+ load_configs
12
+ load_listeners
13
+ end
14
+
15
+ def start_all
16
+ $LOGGER.info "Starting all jobs sequentially"
17
+ $GLOBAL["jobs"].each do |name, config|
18
+ start name
19
+ end
20
+ end
21
+
22
+ def verify_all
23
+ $LOGGER.info "Verifying all jobs sequentially"
24
+ $GLOBAL["jobs"].each do |name, config|
25
+ verify name
26
+ end
27
+ end
28
+
29
+ def start job
30
+ $LOGGER.progname = job
31
+ $LOGGER.info "Started backup"
32
+
33
+ config = _load_config(job)
34
+
35
+ Events.fire :start, {
36
+ job: job,
37
+ config: config,
38
+ }
39
+
40
+ details = _backup(job, config)
41
+
42
+ Events.fire :done, {
43
+ type: :backup,
44
+ job: job,
45
+ config: config,
46
+ details: details,
47
+ }
48
+
49
+ $LOGGER.info "Completed backup"
50
+ end
51
+
52
+ def verify job
53
+ $LOGGER.progname = job
54
+ $LOGGER.info "Started verification"
55
+
56
+ config = _load_config(job)
57
+
58
+ Events.fire :start, {
59
+ job: job,
60
+ config: config,
61
+ }
62
+
63
+ details = _verify(job, config)
64
+
65
+ Events.fire :done, {
66
+ type: :verify,
67
+ job: job,
68
+ config: config,
69
+ details: details,
70
+ }
71
+
72
+ $LOGGER.info "Completed verification"
73
+ end
74
+
75
+ def show job = nil
76
+ return _load_config(job) if job
77
+ $GLOBAL
78
+ end
79
+
80
+ private
81
+ def _load_config job
82
+ config = $GLOBAL["jobs"][job]
83
+ raise "Job #{job} is not defined." unless config
84
+ raise "Job #{job} is not enabled." unless config.fetch("enabled", true)
85
+ raise "Job #{job} has no type." unless config["type"]
86
+
87
+ config
88
+ end
89
+
90
+ def _backup job, config
91
+ type = config["type"].capitalize
92
+ klass = class_for_name("#{ADAPTER_PREFIX}#{type}")
93
+ raise RuntimeError, "Could not load the #{type} adapter." unless klass
94
+ adapter = klass.new(config)
95
+ details = adapter.run()
96
+
97
+ ### Test the verify step immediately after
98
+ # details["verify"] = _verify(job, config)
99
+
100
+ details
101
+ end
102
+
103
+ def _verify job, config
104
+ type = config["type"].capitalize
105
+ klass = class_for_name("#{VERIFY_PREFIX}#{type}")
106
+ raise RuntimeError, "Could not load the #{type} verify." unless klass
107
+ adapter = klass.new(config)
108
+ adapter.verify()
109
+ end
110
+
111
+ def class_for_name name
112
+ name.split("::").inject(Object) { |o, c| o.const_get c }
113
+ end
114
+
115
+ def load_listeners
116
+ Dir["#{File.dirname(__FILE__)}/listeners/**/*.rb"].each do |file|
117
+ require file
118
+
119
+ name = file.gsub("#{File.dirname(__FILE__)}/listeners/", "")
120
+ name = name[0..-4]
121
+ full = ""
122
+ name.split("/").each do |v|
123
+ full += "::#{v.capitalize}"
124
+ end
125
+
126
+ listeners = $GLOBAL["backups"].fetch("listeners", {})
127
+ # search = name.gsub("/", ".")
128
+ search = File.basename(name)
129
+ config = listeners.fetch(search, {})
130
+
131
+ klass = class_for_name("Backups::Listeners#{full}")
132
+ listn = klass.new(config)
133
+ end
134
+ end
135
+
136
+ end
137
+ end
@@ -0,0 +1,80 @@
1
+ module Backups
2
+ module Stats
3
+ module Mysql
4
+
5
+ include ::Backups::Driver::Mysql
6
+
7
+ def get_server_tables(databases = nil)
8
+ query = <<-EOS
9
+ SELECT
10
+ TABLE_SCHEMA `database`,
11
+ TABLE_NAME `table`
12
+ FROM
13
+ INFORMATION_SCHEMA.TABLES
14
+ WHERE
15
+ TABLE_SCHEMA NOT IN ('#{get_excluded_schemas.join("', '")}')
16
+ EOS
17
+
18
+ query += "AND TABLE_SCHEMA IN ('#{databases.join("', '")}')" if databases
19
+ query += "ORDER BY TABLE_SCHEMA ASC"
20
+
21
+ get_results(query)
22
+ end
23
+
24
+ def get_database_tables(database)
25
+ query = <<-EOS
26
+ SELECT
27
+ TABLE_NAME `table`
28
+ FROM
29
+ INFORMATION_SCHEMA.TABLES
30
+ WHERE
31
+ TABLE_SCHEMA = '#{database}'
32
+ EOS
33
+
34
+ get_results(query)
35
+ end
36
+
37
+ def get_database_stats(database)
38
+ tables = get_database_tables(database)
39
+ stats = {}
40
+ tables.each do |item|
41
+ table = item['table']
42
+
43
+ stats[table] = get_table_stats(database, table)
44
+ $LOGGER.debug "Writing stats for table #{database}.#{table}: #{stats[table]}"
45
+ end
46
+
47
+ return stats
48
+ end
49
+
50
+ def get_excluded_schemas
51
+ ["information_schema", "performance_schema", "mysql"]
52
+ end
53
+
54
+ def get_database_names
55
+ query = <<-EOS
56
+ SELECT
57
+ SCHEMA_NAME `database`
58
+ FROM
59
+ INFORMATION_SCHEMA.SCHEMATA
60
+ WHERE
61
+ SCHEMA_NAME NOT IN ('#{get_excluded_schemas.join("', '")}')
62
+ EOS
63
+
64
+ get_results(query)
65
+ end
66
+
67
+ def get_table_stats(database, table)
68
+ query = <<-EOS
69
+ SELECT
70
+ COUNT(*) `rows`
71
+ FROM
72
+ `#{database}`.`#{table}`
73
+ EOS
74
+
75
+ get_result(query)
76
+ end
77
+
78
+ end
79
+ end
80
+ end