datadog_backup 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/workflows/rspec_and_release.yml +54 -0
- data/.gitignore +119 -0
- data/CHANGELOG.md +80 -0
- data/CODE_OF_CONDUCT.md +76 -0
- data/Gemfile +12 -0
- data/Guardfile +44 -0
- data/LICENSE +21 -0
- data/README.md +57 -0
- data/bin/datadog_backup +73 -0
- data/datadog_backup.gemspec +31 -0
- data/example/.github/workflows/backup.yml +31 -0
- data/example/.gitignore +1 -0
- data/example/Gemfile +3 -0
- data/example/README.md +17 -0
- data/lib/datadog_backup.rb +19 -0
- data/lib/datadog_backup/cli.rb +141 -0
- data/lib/datadog_backup/core.rb +113 -0
- data/lib/datadog_backup/dashboards.rb +50 -0
- data/lib/datadog_backup/local_filesystem.rb +86 -0
- data/lib/datadog_backup/monitors.rb +42 -0
- data/lib/datadog_backup/options.rb +46 -0
- data/lib/datadog_backup/thread_pool.rb +30 -0
- data/lib/datadog_backup/version.rb +5 -0
- data/release.config.js +43 -0
- data/spec/datadog_backup/cli_spec.rb +117 -0
- data/spec/datadog_backup/core_spec.rb +107 -0
- data/spec/datadog_backup/dashboards_spec.rb +102 -0
- data/spec/datadog_backup/local_filesystem_spec.rb +173 -0
- data/spec/datadog_backup/monitors_spec.rb +119 -0
- data/spec/datadog_backup_bin_spec.rb +59 -0
- data/spec/datadog_backup_spec.rb +6 -0
- data/spec/spec_helper.rb +37 -0
- metadata +199 -0
data/bin/datadog_backup
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '../lib')
|
5
|
+
|
6
|
+
require 'logger'
|
7
|
+
require 'optparse'
|
8
|
+
|
9
|
+
require 'datadog_backup'
|
10
|
+
require 'dogapi'
|
11
|
+
|
12
|
+
$stdout.sync = $stderr.sync = true
|
13
|
+
LOGGER = Logger.new($stderr) unless defined?(LOGGER)
|
14
|
+
LOGGER.level = Logger::INFO
|
15
|
+
|
16
|
+
##
|
17
|
+
# Default parameters
|
18
|
+
@options = {
|
19
|
+
action: nil,
|
20
|
+
datadog_api_key: ENV.fetch('DATADOG_API_KEY'),
|
21
|
+
datadog_app_key: ENV.fetch('DATADOG_APP_KEY'),
|
22
|
+
backup_dir: File.join(ENV.fetch('PWD'), 'backup'),
|
23
|
+
diff_format: :color,
|
24
|
+
resources: [DatadogBackup::Dashboards, DatadogBackup::Monitors],
|
25
|
+
output_format: :yaml,
|
26
|
+
logger: LOGGER
|
27
|
+
}
|
28
|
+
|
29
|
+
def prereqs
|
30
|
+
ARGV << '--help' if ARGV.empty?
|
31
|
+
|
32
|
+
options = OptionParser.new do |opts|
|
33
|
+
opts.banner = "Usage: #{File.basename($PROGRAM_NAME)} <backup|diffs>"
|
34
|
+
opts.separator ''
|
35
|
+
opts.on_tail('-h', '--help', 'Show this message') do
|
36
|
+
puts opts
|
37
|
+
exit 0
|
38
|
+
end
|
39
|
+
opts.on('--debug', 'debug logging') do
|
40
|
+
LOGGER.level = Logger::DEBUG
|
41
|
+
end
|
42
|
+
opts.on('--backup-dir PATH', '`backup` by default') do |path|
|
43
|
+
@options[:backup_dir] = path
|
44
|
+
end
|
45
|
+
opts.on('--monitors-only') do
|
46
|
+
@options[:resources] = [DatadogBackup::Monitors]
|
47
|
+
end
|
48
|
+
opts.on('--dashboards-only') do
|
49
|
+
@options[:resources] = [DatadogBackup::Dashboards]
|
50
|
+
end
|
51
|
+
opts.on('--json', 'format backups as JSON instead of YAML. Does not impact `diffs` nor `restore`, but do not mix.') do
|
52
|
+
@options[:output_format] = :json
|
53
|
+
end
|
54
|
+
opts.on('--no-color', 'removes colored output from diff format') do
|
55
|
+
@options[:diff_format] = nil
|
56
|
+
end
|
57
|
+
opts.on('--diff-format FORMAT', 'one of `color`, `html_simple`, `html`') do |format|
|
58
|
+
@options[:diff_format] = format.to_sym
|
59
|
+
end
|
60
|
+
end
|
61
|
+
options.parse!
|
62
|
+
|
63
|
+
@options[:action] = ARGV.first
|
64
|
+
if %w[backup diffs restore].include?(@options[:action])
|
65
|
+
else
|
66
|
+
puts options
|
67
|
+
exit 0
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
prereqs
|
72
|
+
|
73
|
+
DatadogBackup::Cli.new(@options).run!
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
4
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
+
require 'datadog_backup/version'
|
6
|
+
|
7
|
+
Gem::Specification.new do |spec|
|
8
|
+
spec.name = 'datadog_backup'
|
9
|
+
spec.version = DatadogBackup::VERSION
|
10
|
+
spec.authors = ['Kamran Farhadi', 'Jim Park']
|
11
|
+
spec.email = ['kamranf@scribd.com', 'jimp@scribd.com']
|
12
|
+
spec.summary = 'A utility to backup and restore Datadog accounts'
|
13
|
+
spec.description = 'A utility to backup and restore Datadog accounts'
|
14
|
+
spec.homepage = 'https://github.com/scribd/datadog_backup'
|
15
|
+
spec.license = 'MIT'
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0")
|
18
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
|
+
spec.test_files = spec.files.grep(%r{^spec/})
|
20
|
+
spec.require_paths = ['lib']
|
21
|
+
|
22
|
+
spec.add_dependency 'amazing_print', '1.2.1'
|
23
|
+
spec.add_dependency 'concurrent-ruby', '1.1.6'
|
24
|
+
spec.add_dependency 'concurrent-ruby-edge', '0.6.0'
|
25
|
+
spec.add_dependency 'deepsort', '0.4.5'
|
26
|
+
spec.add_dependency 'diffy', '3.3.0'
|
27
|
+
spec.add_dependency 'dogapi', '1.41.0'
|
28
|
+
|
29
|
+
spec.add_development_dependency 'bundler'
|
30
|
+
spec.add_development_dependency 'pry'
|
31
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
name: backup
|
2
|
+
|
3
|
+
on:
|
4
|
+
schedule:
|
5
|
+
- cron: "0 * * * *"
|
6
|
+
workflow_dispatch:
|
7
|
+
|
8
|
+
jobs:
|
9
|
+
backup:
|
10
|
+
runs-on: ubuntu-latest
|
11
|
+
|
12
|
+
steps:
|
13
|
+
- uses: actions/checkout@v2
|
14
|
+
- name: Set up Ruby 2.7.1
|
15
|
+
uses: actions/setup-ruby@v1
|
16
|
+
with:
|
17
|
+
ruby-version: 2.7.1
|
18
|
+
- name: perform backup
|
19
|
+
env:
|
20
|
+
DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
|
21
|
+
DATADOG_APP_KEY: ${{ secrets.DATADOG_APP_KEY }}
|
22
|
+
run: |
|
23
|
+
gem install --no-document bundler
|
24
|
+
bundle install --jobs 4 --retry 3
|
25
|
+
bundle exec datadog_backup backup
|
26
|
+
- name: commit changes
|
27
|
+
uses: stefanzweifel/git-auto-commit-action@v4
|
28
|
+
with:
|
29
|
+
commit_message: "Changes as of run: ${{ github.run_id }}"
|
30
|
+
file_pattern: backup/
|
31
|
+
repository: .
|
data/example/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
.bundle/
|
data/example/Gemfile
ADDED
data/example/README.md
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# Example-datadog-backups
|
2
|
+
Dashboards and monitors are backed up here on an hourly basis.
|
3
|
+
|
4
|
+
Github Actions uses the [datadog_backup gem](https://github.com/scribd/datadog_backup) in order to sync the latest copy of what's in Datadog.
|
5
|
+
|
6
|
+
## Performing edits to Datadog monitors and dashboards
|
7
|
+
1. Do it in the Datadog UI
|
8
|
+
2. At the top of the hour, the changes will be recorded here.
|
9
|
+
3. If you're in a rush, click on "Run workflow" from the Github Actions workflow menus
|
10
|
+
|
11
|
+
## Performing bulk edits to Datadog monitors and dashboards
|
12
|
+
1. Clone this repo
|
13
|
+
2. `bundle install`
|
14
|
+
3. `bundle exec datadog_backup backup` to download the latest changes in Datadog.
|
15
|
+
4. Make your changes locally.
|
16
|
+
5. `bundle exec datadog_backup restore` to apply your changes.
|
17
|
+
6. Review each change and apply it (r), or download the latest copy from Datadog (d).
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'concurrent'
|
4
|
+
require 'concurrent-edge'
|
5
|
+
|
6
|
+
require 'dogapi'
|
7
|
+
|
8
|
+
require_relative 'datadog_backup/local_filesystem'
|
9
|
+
require_relative 'datadog_backup/options'
|
10
|
+
|
11
|
+
require_relative 'datadog_backup/cli'
|
12
|
+
require_relative 'datadog_backup/core'
|
13
|
+
require_relative 'datadog_backup/dashboards'
|
14
|
+
require_relative 'datadog_backup/monitors'
|
15
|
+
require_relative 'datadog_backup/thread_pool'
|
16
|
+
require_relative 'datadog_backup/version'
|
17
|
+
|
18
|
+
module DatadogBackup
|
19
|
+
end
|
@@ -0,0 +1,141 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
require 'logger'
|
5
|
+
require 'amazing_print'
|
6
|
+
|
7
|
+
module DatadogBackup
|
8
|
+
class Cli
|
9
|
+
include ::DatadogBackup::Options
|
10
|
+
|
11
|
+
def all_diff_futures
|
12
|
+
logger.info("Starting diffs on #{::DatadogBackup::ThreadPool::TPOOL.max_length} threads")
|
13
|
+
any_resource_instance
|
14
|
+
.all_file_ids_for_selected_resources
|
15
|
+
.map do |id|
|
16
|
+
Concurrent::Promises.future_on(::DatadogBackup::ThreadPool::TPOOL, id) do |id|
|
17
|
+
[id, getdiff(id)]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def any_resource_instance
|
23
|
+
resource_instances.first
|
24
|
+
end
|
25
|
+
|
26
|
+
def backup
|
27
|
+
resource_instances.each(&:purge)
|
28
|
+
resource_instances.each(&:backup)
|
29
|
+
any_resource_instance.all_files
|
30
|
+
end
|
31
|
+
|
32
|
+
def initialize_client
|
33
|
+
@options[:client] ||= Dogapi::Client.new(
|
34
|
+
datadog_api_key,
|
35
|
+
datadog_app_key
|
36
|
+
)
|
37
|
+
end
|
38
|
+
|
39
|
+
def definitive_resource_instance(id)
|
40
|
+
matching_resource_instance(any_resource_instance.class_from_id(id))
|
41
|
+
end
|
42
|
+
|
43
|
+
def diffs
|
44
|
+
futures = all_diff_futures
|
45
|
+
::DatadogBackup::ThreadPool.watcher(logger).join
|
46
|
+
|
47
|
+
format_diff_output(
|
48
|
+
Concurrent::Promises
|
49
|
+
.zip(*futures)
|
50
|
+
.value!
|
51
|
+
.reject { |_k, v| v.nil? }
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
def getdiff(id)
|
56
|
+
result = definitive_resource_instance(id).diff(id)
|
57
|
+
case result
|
58
|
+
when ''
|
59
|
+
nil
|
60
|
+
when "\n"
|
61
|
+
nil
|
62
|
+
when '<div class="diff"></div>'
|
63
|
+
nil
|
64
|
+
else
|
65
|
+
result
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def format_diff_output(diff_output)
|
70
|
+
case diff_format
|
71
|
+
when nil, :color
|
72
|
+
diff_output.map do |id, diff|
|
73
|
+
" ---\n id: #{id}\n#{diff}"
|
74
|
+
end.join("\n")
|
75
|
+
when :html
|
76
|
+
'<html><head><style>' +
|
77
|
+
Diffy::CSS +
|
78
|
+
'</style></head><body>' +
|
79
|
+
diff_output.map do |id, diff|
|
80
|
+
"<br><br> ---<br><strong> id: #{id}</strong><br>" + diff
|
81
|
+
end.join('<br>') +
|
82
|
+
'</body></html>'
|
83
|
+
else
|
84
|
+
raise 'Unexpected diff_format.'
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def initialize(options)
|
89
|
+
@options = options
|
90
|
+
initialize_client
|
91
|
+
end
|
92
|
+
|
93
|
+
def matching_resource_instance(klass)
|
94
|
+
resource_instances.select { |resource_instance| resource_instance.class == klass }.first
|
95
|
+
end
|
96
|
+
|
97
|
+
def resource_instances
|
98
|
+
@resource_instances ||= resources.map do |resource|
|
99
|
+
resource.new(@options)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def restore
|
104
|
+
futures = all_diff_futures
|
105
|
+
watcher = ::DatadogBackup::ThreadPool.watcher(logger)
|
106
|
+
|
107
|
+
futures.each do |future|
|
108
|
+
id, diff = *future.value!
|
109
|
+
next unless diff
|
110
|
+
|
111
|
+
puts '--------------------------------------------------------------------------------'
|
112
|
+
puts format_diff_output([id, diff])
|
113
|
+
puts '(r)estore to Datadog, overwrite local changes and (d)ownload, (s)kip, or (q)uit?'
|
114
|
+
response = $stdin.gets.chomp
|
115
|
+
case response
|
116
|
+
when 'q'
|
117
|
+
exit
|
118
|
+
when 'r'
|
119
|
+
puts "Restoring #{id} to Datadog."
|
120
|
+
definitive_resource_instance(id).update_with_200(id, definitive_resource_instance(id).load_from_file_by_id(id))
|
121
|
+
when 'd'
|
122
|
+
puts "Downloading #{id} from Datadog."
|
123
|
+
definitive_resource_instance(id).get_and_write_file(id)
|
124
|
+
when 's'
|
125
|
+
next
|
126
|
+
else
|
127
|
+
puts 'Invalid response, please try again.'
|
128
|
+
response = $stdin.gets.chomp
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
watcher.join if watcher.status
|
133
|
+
end
|
134
|
+
|
135
|
+
def run!
|
136
|
+
puts(send(action.to_sym))
|
137
|
+
rescue SystemExit, Interrupt
|
138
|
+
::DatadogBackup::ThreadPool.shutdown(logger)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'diffy'
|
4
|
+
require 'deepsort'
|
5
|
+
|
6
|
+
module DatadogBackup
|
7
|
+
class Core
|
8
|
+
include ::DatadogBackup::LocalFilesystem
|
9
|
+
include ::DatadogBackup::Options
|
10
|
+
|
11
|
+
def api_service
|
12
|
+
raise 'subclass is expected to implement #api_service'
|
13
|
+
end
|
14
|
+
|
15
|
+
def api_version
|
16
|
+
raise 'subclass is expected to implement #api_version'
|
17
|
+
end
|
18
|
+
|
19
|
+
def api_resource_name
|
20
|
+
raise 'subclass is expected to implement #api_resource_name'
|
21
|
+
end
|
22
|
+
|
23
|
+
def backup
|
24
|
+
raise 'subclass is expected to implement #backup'
|
25
|
+
end
|
26
|
+
|
27
|
+
# Calls out to Datadog and checks for a '200' response
|
28
|
+
def client_with_200(method, *id)
|
29
|
+
max_retries = 6
|
30
|
+
retries ||= 0
|
31
|
+
|
32
|
+
response = client.send(method, *id)
|
33
|
+
|
34
|
+
# logger.debug response
|
35
|
+
raise "Method #{method} failed with error #{response}" unless response[0] == '200'
|
36
|
+
|
37
|
+
response[1]
|
38
|
+
rescue ::Net::OpenTimeout => e
|
39
|
+
if (retries += 1) <= max_retries
|
40
|
+
sleep(0.1 * retries**5) # 0.1, 3.2, 24.3, 102.4 seconds per retry
|
41
|
+
retry
|
42
|
+
else
|
43
|
+
raise "Method #{method} failed with error #{e.message}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Returns the diffy diff.
|
48
|
+
# Optionally, supply an array of keys to remove from comparison
|
49
|
+
def diff(id, banlist = [])
|
50
|
+
current = except(get_by_id(id), banlist).deep_sort.to_yaml
|
51
|
+
filesystem = except(load_from_file_by_id(id), banlist).deep_sort.to_yaml
|
52
|
+
result = ::Diffy::Diff.new(current, filesystem, include_plus_and_minus_in_html: true).to_s(diff_format)
|
53
|
+
logger.debug("Compared ID #{id} and found #{result}")
|
54
|
+
result
|
55
|
+
end
|
56
|
+
|
57
|
+
# Returns a hash with banlist elements removed
|
58
|
+
def except(hash, banlist)
|
59
|
+
hash.tap do # tap returns self
|
60
|
+
banlist.each do |key|
|
61
|
+
hash.delete(key) # delete returns the value at the deleted key, hence the tap wrapper
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def get_and_write_file(id)
|
67
|
+
write_file(dump(get_by_id(id)), filename(id))
|
68
|
+
end
|
69
|
+
|
70
|
+
def get_by_id(_id)
|
71
|
+
raise 'subclass is expected to implement #get_by_id(id)'
|
72
|
+
end
|
73
|
+
|
74
|
+
def initialize(options)
|
75
|
+
@options = options
|
76
|
+
::FileUtils.mkdir_p(mydir)
|
77
|
+
end
|
78
|
+
|
79
|
+
def myclass
|
80
|
+
self.class.to_s.split(':').last.downcase
|
81
|
+
end
|
82
|
+
|
83
|
+
def restore
|
84
|
+
raise 'subclass is expected to implement #restore'
|
85
|
+
end
|
86
|
+
|
87
|
+
def update(id, body)
|
88
|
+
api_service.request(Net::HTTP::Put, "/api/#{api_version}/#{api_resource_name}/#{id}", nil, body, true)
|
89
|
+
end
|
90
|
+
|
91
|
+
# Calls out to Datadog and checks for a '200' response
|
92
|
+
def update_with_200(id, body)
|
93
|
+
max_retries = 6
|
94
|
+
retries ||= 0
|
95
|
+
|
96
|
+
response = update(id, body)
|
97
|
+
|
98
|
+
# logger.debug response
|
99
|
+
raise "Update failed with error #{response}" unless response[0] == '200'
|
100
|
+
|
101
|
+
logger.warn "Successfully restored #{id} to datadog."
|
102
|
+
|
103
|
+
response[1]
|
104
|
+
rescue ::Net::OpenTimeout => e
|
105
|
+
if (retries += 1) <= max_retries
|
106
|
+
sleep(0.1 * retries**5) # 0.1, 3.2, 24.3, 102.4 seconds per retry
|
107
|
+
retry
|
108
|
+
else
|
109
|
+
raise "Update failed with error #{e.message}"
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DatadogBackup
|
4
|
+
class Dashboards < Core
|
5
|
+
BANLIST = %w[modified_at url]
|
6
|
+
|
7
|
+
def all_boards
|
8
|
+
client_with_200(:get_all_boards).fetch('dashboards')
|
9
|
+
end
|
10
|
+
|
11
|
+
def api_service
|
12
|
+
# The underlying class from Dogapi that talks to datadog
|
13
|
+
client.instance_variable_get(:@dashboard_service)
|
14
|
+
end
|
15
|
+
|
16
|
+
def api_version
|
17
|
+
'v1'
|
18
|
+
end
|
19
|
+
|
20
|
+
def api_resource_name
|
21
|
+
'dashboard'
|
22
|
+
end
|
23
|
+
|
24
|
+
def backup
|
25
|
+
logger.info("Starting diffs on #{::DatadogBackup::ThreadPool::TPOOL.max_length} threads")
|
26
|
+
|
27
|
+
futures = all_boards.map do |board|
|
28
|
+
Concurrent::Promises.future_on(::DatadogBackup::ThreadPool::TPOOL, board) do |board|
|
29
|
+
id = board['id']
|
30
|
+
get_and_write_file(id)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
watcher = ::DatadogBackup::ThreadPool.watcher(logger)
|
35
|
+
watcher.join if watcher.status
|
36
|
+
|
37
|
+
Concurrent::Promises.zip(*futures).value!
|
38
|
+
end
|
39
|
+
|
40
|
+
def diff(id)
|
41
|
+
super(id, BANLIST)
|
42
|
+
end
|
43
|
+
|
44
|
+
def get_by_id(id)
|
45
|
+
except(client_with_200(:get_board, id), BANLIST)
|
46
|
+
end
|
47
|
+
|
48
|
+
def restore!; end
|
49
|
+
end
|
50
|
+
end
|