datadog_backup 2.0.0 → 3.0.0.alpha.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -0
- data/Gemfile.lock +23 -12
- data/README.md +21 -9
- data/bin/datadog_backup +14 -17
- data/datadog_backup.gemspec +9 -6
- data/example/.github/workflows/backup.yml +2 -2
- data/lib/datadog_backup/cli.rb +4 -13
- data/lib/datadog_backup/core.rb +55 -38
- data/lib/datadog_backup/dashboards.rb +7 -11
- data/lib/datadog_backup/deprecations.rb +11 -0
- data/lib/datadog_backup/local_filesystem.rb +1 -1
- data/lib/datadog_backup/monitors.rb +0 -5
- data/lib/datadog_backup/options.rb +0 -16
- data/lib/datadog_backup/thread_pool.rb +5 -5
- data/lib/datadog_backup/version.rb +1 -1
- data/lib/datadog_backup.rb +6 -2
- data/release.config.js +2 -1
- data/spec/datadog_backup/cli_spec.rb +13 -57
- data/spec/datadog_backup/core_spec.rb +29 -63
- data/spec/datadog_backup/dashboards_spec.rb +15 -16
- data/spec/datadog_backup/deprecations_spec.rb +36 -0
- data/spec/datadog_backup/local_filesystem_spec.rb +2 -7
- data/spec/datadog_backup/monitors_spec.rb +17 -16
- data/spec/datadog_backup_bin_spec.rb +5 -5
- data/spec/spec_helper.rb +7 -6
- metadata +57 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 75f5659461aa4a52626f34a4094174cc899e600695e512c9a8f9b613741cda54
|
4
|
+
data.tar.gz: fc74c95bdbadf48eebb12d27648f365c3e83dfe43f51df5e5f3b41e095b6162f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a199e84cdee10ad8bfd0274dda29cdf67b540451748a8a096eafa166ff5f8063bed16dc93318063fefe085f015c85dbc6f93af202af991776f2ef5219e0e3a59
|
7
|
+
data.tar.gz: be2034d46ec881f89338a2c54880a6f39fd1946288e352a68bd1fcc8338b43066574650d373d9401a35203c95897775dc26773abbd17ba4b597b268fb6396622
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,32 @@
|
|
1
|
+
# [3.0.0-alpha.1](https://github.com/scribd/datadog_backup/compare/v2.0.2...v3.0.0-alpha.1) (2022-08-24)
|
2
|
+
|
3
|
+
|
4
|
+
* feat!: release 3.0 ([d09d9e6](https://github.com/scribd/datadog_backup/commit/d09d9e6c845edb35c49cbb19ec6b35878304a078))
|
5
|
+
|
6
|
+
|
7
|
+
### BREAKING CHANGES
|
8
|
+
|
9
|
+
* DATADOG_API_KEY and DATADOG_APP_KEY are no longer the environment variables used to authenticate to Datadog. Instead, set the environment variables DD_API_KEY and DD_APP_KEY.
|
10
|
+
* ruby 2.6 is no longer supported. Please upgrade to ruby 2.7 or higher.
|
11
|
+
* The options `--ssh` and `--ssshh` are no longer supported. Instead, please use `--quiet` to supress logging. `--debug` remains supported.
|
12
|
+
* The environment variable `DATADOG_HOST` is no longer supported. Instead, please use `DD_SITE_URL`.
|
13
|
+
|
14
|
+
refactor: The legacy [dogapi-rb ](https://github.com/DataDog/dogapi-rb) gem is replaced with [faraday](https://lostisland.github.io/faraday/). The [official client library](https://github.com/DataDog/datadog-api-client-ruby) was considered, but was not adopted as I had a hard time grok-ing it.
|
15
|
+
|
16
|
+
## [2.0.2](https://github.com/scribd/datadog_backup/compare/v2.0.1...v2.0.2) (2022-08-11)
|
17
|
+
|
18
|
+
|
19
|
+
### Bug Fixes
|
20
|
+
|
21
|
+
* Deprecate Ruby 2.6 and Drop support for Ruby 2.5 ([#132](https://github.com/scribd/datadog_backup/issues/132)) ([432cb2c](https://github.com/scribd/datadog_backup/commit/432cb2c0d8b12d89aef81cf35597aa90f77407eb))
|
22
|
+
|
23
|
+
## [2.0.1](https://github.com/scribd/datadog_backup/compare/v2.0.0...v2.0.1) (2022-08-11)
|
24
|
+
|
25
|
+
|
26
|
+
### Bug Fixes
|
27
|
+
|
28
|
+
* include version.rb in release commit ([#130](https://github.com/scribd/datadog_backup/issues/130)) ([f8df6cc](https://github.com/scribd/datadog_backup/commit/f8df6cc48ac9a3521c3c98dfa2c325f96801d001))
|
29
|
+
|
1
30
|
# [2.0.0](https://github.com/scribd/datadog_backup/compare/v1.1.4...v2.0.0) (2022-08-09)
|
2
31
|
|
3
32
|
|
data/Gemfile.lock
CHANGED
@@ -1,25 +1,31 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
datadog_backup (0.
|
5
|
-
amazing_print
|
6
|
-
concurrent-ruby
|
7
|
-
deepsort
|
8
|
-
diffy
|
9
|
-
|
4
|
+
datadog_backup (2.0.2)
|
5
|
+
amazing_print
|
6
|
+
concurrent-ruby
|
7
|
+
deepsort
|
8
|
+
diffy
|
9
|
+
faraday
|
10
|
+
faraday-retry
|
10
11
|
|
11
12
|
GEM
|
12
13
|
remote: https://rubygems.org/
|
13
14
|
specs:
|
14
15
|
amazing_print (1.4.0)
|
15
16
|
ast (2.4.2)
|
17
|
+
byebug (11.1.3)
|
16
18
|
coderay (1.1.3)
|
17
19
|
concurrent-ruby (1.1.10)
|
18
20
|
deepsort (0.4.5)
|
19
21
|
diff-lcs (1.5.0)
|
20
22
|
diffy (3.4.2)
|
21
|
-
|
22
|
-
|
23
|
+
faraday (2.5.2)
|
24
|
+
faraday-net_http (>= 2.0, < 3.1)
|
25
|
+
ruby2_keywords (>= 0.0.4)
|
26
|
+
faraday-net_http (3.0.0)
|
27
|
+
faraday-retry (2.0.0)
|
28
|
+
faraday (~> 2.0)
|
23
29
|
ffi (1.15.5)
|
24
30
|
formatador (1.1.0)
|
25
31
|
guard (2.18.0)
|
@@ -42,7 +48,6 @@ GEM
|
|
42
48
|
rb-inotify (~> 0.9, >= 0.9.10)
|
43
49
|
lumberjack (1.2.8)
|
44
50
|
method_source (1.0.0)
|
45
|
-
multi_json (1.15.0)
|
46
51
|
nenv (0.3.0)
|
47
52
|
notiffany (0.1.3)
|
48
53
|
nenv (~> 0.1)
|
@@ -53,6 +58,9 @@ GEM
|
|
53
58
|
pry (0.14.1)
|
54
59
|
coderay (~> 1.1)
|
55
60
|
method_source (~> 1.0)
|
61
|
+
pry-byebug (3.10.1)
|
62
|
+
byebug (~> 11.0)
|
63
|
+
pry (>= 0.13, < 0.15)
|
56
64
|
rainbow (3.1.1)
|
57
65
|
rb-fsevent (0.11.1)
|
58
66
|
rb-inotify (0.10.1)
|
@@ -72,14 +80,14 @@ GEM
|
|
72
80
|
diff-lcs (>= 1.2.0, < 2.0)
|
73
81
|
rspec-support (~> 3.11.0)
|
74
82
|
rspec-support (3.11.0)
|
75
|
-
rubocop (1.
|
83
|
+
rubocop (1.35.1)
|
76
84
|
json (~> 2.3)
|
77
85
|
parallel (~> 1.10)
|
78
86
|
parser (>= 3.1.2.1)
|
79
87
|
rainbow (>= 2.2.2, < 4.0)
|
80
88
|
regexp_parser (>= 1.8, < 3.0)
|
81
89
|
rexml (>= 3.2.5, < 4.0)
|
82
|
-
rubocop-ast (>= 1.20.
|
90
|
+
rubocop-ast (>= 1.20.1, < 2.0)
|
83
91
|
ruby-progressbar (~> 1.7)
|
84
92
|
unicode-display_width (>= 1.4.0, < 3.0)
|
85
93
|
rubocop-ast (1.21.0)
|
@@ -87,21 +95,24 @@ GEM
|
|
87
95
|
rubocop-rspec (2.12.1)
|
88
96
|
rubocop (~> 1.31)
|
89
97
|
ruby-progressbar (1.11.0)
|
98
|
+
ruby2_keywords (0.0.5)
|
90
99
|
shellany (0.0.1)
|
91
100
|
thor (1.2.1)
|
92
101
|
unicode-display_width (2.2.0)
|
93
102
|
|
94
103
|
PLATFORMS
|
95
104
|
ruby
|
105
|
+
x86_64-linux
|
96
106
|
|
97
107
|
DEPENDENCIES
|
98
108
|
bundler
|
99
109
|
datadog_backup!
|
100
110
|
guard-rspec
|
101
111
|
pry
|
112
|
+
pry-byebug
|
102
113
|
rspec
|
103
114
|
rubocop
|
104
115
|
rubocop-rspec
|
105
116
|
|
106
117
|
BUNDLED WITH
|
107
|
-
2.3.
|
118
|
+
2.3.20
|
data/README.md
CHANGED
@@ -11,6 +11,20 @@ Currently supports
|
|
11
11
|
|
12
12
|
Additional features may be built out over time.
|
13
13
|
|
14
|
+
# v3 Migration
|
15
|
+
|
16
|
+
## Breaking Changes
|
17
|
+
v3 is a backwards incompatible change.
|
18
|
+
|
19
|
+
- [] DATADOG_API_KEY and DATADOG_APP_KEY are no longer the environment variables used to authenticate to Datadog. Instead, set the environment variables DD_API_KEY and DD_APP_KEY.
|
20
|
+
- [ ] ruby 2.6 is no longer supported. Please upgrade to ruby 2.7 or higher.
|
21
|
+
- [ ] The options `--ssh` and `--ssshh` are no longer supported. Instead, please use `--quiet` to supress logging. `--debug` remains supported.
|
22
|
+
- [ ] The environment variable `DATADOG_HOST` is no longer supported. Instead, please use `DD_SITE_URL`.
|
23
|
+
|
24
|
+
## Misc
|
25
|
+
- [ ] The legacy [dogapi-rb ](https://github.com/DataDog/dogapi-rb) gem is replaced with [faraday](https://lostisland.github.io/faraday/). The [official client library](https://github.com/DataDog/datadog-api-client-ruby) was considered, but was not adopted as I had a hard time grok-ing it.
|
26
|
+
|
27
|
+
|
14
28
|
## Installation
|
15
29
|
|
16
30
|
```
|
@@ -22,13 +36,13 @@ gem install datadog_backup
|
|
22
36
|
![demo](images/demo.gif)
|
23
37
|
|
24
38
|
```
|
25
|
-
|
39
|
+
DD_API_KEY=example123 DD_APP_KEY=example123 datadog_backup <backup|diffs|restore> [--backup-dir /path/to/backups] [--debug] [--monitors-only] [--dashboards-only] [--diff-format color|html|html_simple] [--no-color] [--json]
|
26
40
|
```
|
27
41
|
|
28
42
|
```
|
29
43
|
gem install datadog_backup
|
30
|
-
export
|
31
|
-
export
|
44
|
+
export DD_API_KEY=abc123
|
45
|
+
export DD_APP_KEY=abc123
|
32
46
|
|
33
47
|
# Perform backup to `./backup/` using YAML encoding
|
34
48
|
datadog_backup backup
|
@@ -49,8 +63,7 @@ Supply the following parameters in order to customize datadog_backup:
|
|
49
63
|
parameter | description | default
|
50
64
|
---------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------
|
51
65
|
--debug | log debug and above | info
|
52
|
-
--
|
53
|
-
--shhh | log errors and above | info
|
66
|
+
--quiet | only show errors and above | info
|
54
67
|
--backup-dir PATH | path to the directory to backup to or restore from | `./backup/`
|
55
68
|
--monitors-only | only backup monitors | backup monitors and dashboards
|
56
69
|
--dashboards-only | only backup dashboards | backup monitors and dashboards
|
@@ -66,10 +79,9 @@ The following environment variables can be set in order to further customize dat
|
|
66
79
|
|
67
80
|
environment variable | description | default
|
68
81
|
---------------------|--------------------------------------------------------------------------------|--------------------------
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
dd_proxy_https | Same as `http_proxy` | none
|
82
|
+
DD_SITE_URL | Describe the API endpoint to connect to (https://api.datadoghq.eu for example) | https://api.datadoghq.com
|
83
|
+
DD_API_KEY | The API key for the Datadog account | none
|
84
|
+
DD_API_KEY | The Application key for the Datadog account | none
|
73
85
|
|
74
86
|
|
75
87
|
### Usage in a Github repo
|
data/bin/datadog_backup
CHANGED
@@ -5,23 +5,21 @@ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '../lib')
|
|
5
5
|
|
6
6
|
require 'logger'
|
7
7
|
require 'optparse'
|
8
|
-
|
9
|
-
require 'datadog_backup'
|
10
|
-
require 'dogapi'
|
11
|
-
|
12
8
|
$stdout.sync = $stderr.sync = true
|
13
9
|
LOGGER = Logger.new($stderr) unless defined?(LOGGER)
|
14
10
|
LOGGER.level = Logger::INFO
|
15
11
|
|
12
|
+
require 'datadog_backup'
|
13
|
+
|
14
|
+
|
16
15
|
def fatal(message)
|
17
16
|
LOGGER.fatal(message)
|
18
17
|
exit 1
|
19
18
|
end
|
20
19
|
|
21
20
|
def options_valid?(options)
|
22
|
-
%w[backup diffs restore].include?(options[:action])
|
23
|
-
|
24
|
-
options[:datadog_app_key]
|
21
|
+
%w[backup diffs restore].include?(options[:action])
|
22
|
+
%w[DD_API_KEY DD_APP_KEY].all? { |key| ENV[key] }
|
25
23
|
end
|
26
24
|
|
27
25
|
def prereqs(defaults)
|
@@ -30,7 +28,7 @@ def prereqs(defaults)
|
|
30
28
|
result = defaults.dup
|
31
29
|
|
32
30
|
options = OptionParser.new do |opts|
|
33
|
-
opts.banner = "Usage:
|
31
|
+
opts.banner = "Usage: DD_API_KEY=abc123 DD_APP_KEY=abc123 #{File.basename($PROGRAM_NAME)} <backup|diffs|restore>"
|
34
32
|
opts.separator ''
|
35
33
|
opts.on_tail('-h', '--help', 'Show this message') do
|
36
34
|
puts opts
|
@@ -39,10 +37,7 @@ def prereqs(defaults)
|
|
39
37
|
opts.on('--debug', 'log debug and above') do
|
40
38
|
LOGGER.level = Logger::DEBUG
|
41
39
|
end
|
42
|
-
opts.on('--
|
43
|
-
LOGGER.level = Logger::WARN
|
44
|
-
end
|
45
|
-
opts.on('--shhh', 'log errors and above') do
|
40
|
+
opts.on('--quiet', 'log errors and above') do
|
46
41
|
LOGGER.level = Logger::ERROR
|
47
42
|
end
|
48
43
|
opts.on('--backup-dir PATH', '`backup` by default') do |path|
|
@@ -73,7 +68,7 @@ def prereqs(defaults)
|
|
73
68
|
options.parse!
|
74
69
|
|
75
70
|
result[:action] = ARGV.first
|
76
|
-
fatal(options) unless options_valid?(result)
|
71
|
+
fatal(options.banner) unless options_valid?(result)
|
77
72
|
result
|
78
73
|
end
|
79
74
|
|
@@ -81,14 +76,16 @@ end
|
|
81
76
|
# Default parameters
|
82
77
|
defaults = {
|
83
78
|
action: nil,
|
84
|
-
datadog_api_key: ENV.fetch('DATADOG_API_KEY', nil),
|
85
|
-
datadog_app_key: ENV.fetch('DATADOG_APP_KEY', nil),
|
86
79
|
backup_dir: File.join(ENV.fetch('PWD'), 'backup'),
|
87
80
|
diff_format: :color,
|
88
81
|
resources: [DatadogBackup::Dashboards, DatadogBackup::Monitors],
|
89
82
|
output_format: :yaml,
|
90
|
-
force_restore: false
|
91
|
-
logger: LOGGER
|
83
|
+
force_restore: false
|
92
84
|
}
|
93
85
|
|
86
|
+
begin
|
94
87
|
DatadogBackup::Cli.new(prereqs(defaults)).run!
|
88
|
+
rescue => e
|
89
|
+
require 'pry'
|
90
|
+
binding.pry
|
91
|
+
end
|
data/datadog_backup.gemspec
CHANGED
@@ -19,16 +19,19 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.test_files = spec.files.grep(%r{^spec/})
|
20
20
|
spec.require_paths = ['lib']
|
21
21
|
|
22
|
-
spec.required_ruby_version = ['>= 2.
|
22
|
+
spec.required_ruby_version = ['>= 2.7']
|
23
|
+
|
24
|
+
spec.add_dependency 'amazing_print'
|
25
|
+
spec.add_dependency 'concurrent-ruby'
|
26
|
+
spec.add_dependency 'deepsort'
|
27
|
+
spec.add_dependency 'diffy'
|
28
|
+
spec.add_dependency 'faraday'
|
29
|
+
spec.add_dependency 'faraday-retry'
|
23
30
|
|
24
|
-
spec.add_dependency 'amazing_print', '1.4.0'
|
25
|
-
spec.add_dependency 'concurrent-ruby', '1.1.10'
|
26
|
-
spec.add_dependency 'deepsort', '0.4.5'
|
27
|
-
spec.add_dependency 'diffy', '3.4.2'
|
28
|
-
spec.add_dependency 'dogapi', '1.45.0'
|
29
31
|
|
30
32
|
spec.add_development_dependency 'bundler'
|
31
33
|
spec.add_development_dependency 'pry'
|
34
|
+
spec.add_development_dependency 'pry-byebug'
|
32
35
|
spec.add_development_dependency 'guard-rspec'
|
33
36
|
spec.add_development_dependency 'rspec'
|
34
37
|
spec.add_development_dependency 'rubocop'
|
@@ -17,8 +17,8 @@ jobs:
|
|
17
17
|
ruby-version: 2.7.1
|
18
18
|
- name: perform backup
|
19
19
|
env:
|
20
|
-
|
21
|
-
|
20
|
+
DD_API_KEY: ${{ secrets.DD_API_KEY }}
|
21
|
+
DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
|
22
22
|
run: |
|
23
23
|
gem install --no-document bundler
|
24
24
|
bundle install --jobs 4 --retry 3
|
data/lib/datadog_backup/cli.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'optparse'
|
4
|
-
require 'logger'
|
5
4
|
require 'amazing_print'
|
6
5
|
|
7
6
|
module DatadogBackup
|
@@ -9,7 +8,7 @@ module DatadogBackup
|
|
9
8
|
include ::DatadogBackup::Options
|
10
9
|
|
11
10
|
def all_diff_futures
|
12
|
-
|
11
|
+
LOGGER.info("Starting diffs on #{::DatadogBackup::ThreadPool::TPOOL.max_length} threads")
|
13
12
|
any_resource_instance
|
14
13
|
.all_file_ids_for_selected_resources
|
15
14
|
.map do |id|
|
@@ -29,20 +28,13 @@ module DatadogBackup
|
|
29
28
|
any_resource_instance.all_files
|
30
29
|
end
|
31
30
|
|
32
|
-
def initialize_client
|
33
|
-
@options[:client] ||= Dogapi::Client.new(
|
34
|
-
datadog_api_key,
|
35
|
-
datadog_app_key
|
36
|
-
)
|
37
|
-
end
|
38
|
-
|
39
31
|
def definitive_resource_instance(id)
|
40
32
|
matching_resource_instance(any_resource_instance.class_from_id(id))
|
41
33
|
end
|
42
34
|
|
43
35
|
def diffs
|
44
36
|
futures = all_diff_futures
|
45
|
-
::DatadogBackup::ThreadPool.watcher
|
37
|
+
::DatadogBackup::ThreadPool.watcher.join
|
46
38
|
|
47
39
|
format_diff_output(
|
48
40
|
Concurrent::Promises
|
@@ -87,7 +79,6 @@ module DatadogBackup
|
|
87
79
|
|
88
80
|
def initialize(options)
|
89
81
|
@options = options
|
90
|
-
initialize_client
|
91
82
|
end
|
92
83
|
|
93
84
|
def matching_resource_instance(klass)
|
@@ -102,7 +93,7 @@ module DatadogBackup
|
|
102
93
|
|
103
94
|
def restore
|
104
95
|
futures = all_diff_futures
|
105
|
-
watcher = ::DatadogBackup::ThreadPool.watcher
|
96
|
+
watcher = ::DatadogBackup::ThreadPool.watcher
|
106
97
|
|
107
98
|
futures.each do |future|
|
108
99
|
id, diff = *future.value!
|
@@ -138,7 +129,7 @@ module DatadogBackup
|
|
138
129
|
def run!
|
139
130
|
puts(send(action.to_sym))
|
140
131
|
rescue SystemExit, Interrupt
|
141
|
-
::DatadogBackup::ThreadPool.shutdown
|
132
|
+
::DatadogBackup::ThreadPool.shutdown
|
142
133
|
end
|
143
134
|
end
|
144
135
|
end
|
data/lib/datadog_backup/core.rb
CHANGED
@@ -2,14 +2,45 @@
|
|
2
2
|
|
3
3
|
require 'diffy'
|
4
4
|
require 'deepsort'
|
5
|
+
require 'faraday'
|
6
|
+
require 'faraday/retry'
|
7
|
+
|
8
|
+
|
5
9
|
|
6
10
|
module DatadogBackup
|
7
11
|
class Core
|
8
12
|
include ::DatadogBackup::LocalFilesystem
|
9
13
|
include ::DatadogBackup::Options
|
10
14
|
|
15
|
+
@@retry_options = {
|
16
|
+
max: 5,
|
17
|
+
interval: 0.05,
|
18
|
+
interval_randomness: 0.5,
|
19
|
+
backoff_factor: 2
|
20
|
+
}
|
21
|
+
|
11
22
|
def api_service
|
12
|
-
|
23
|
+
conn ||= Faraday.new(
|
24
|
+
url: api_url,
|
25
|
+
headers: {
|
26
|
+
'DD-API-KEY' => ENV.fetch('DD_API_KEY'),
|
27
|
+
'DD-APPLICATION-KEY' => ENV.fetch('DD_APP_KEY')
|
28
|
+
}
|
29
|
+
) do |faraday|
|
30
|
+
faraday.request :json
|
31
|
+
faraday.request :retry, @@retry_options
|
32
|
+
faraday.response(:logger, LOGGER, {headers: true, bodies: LOGGER.debug?, log_level: :debug}) do | logger |
|
33
|
+
logger.filter(/(DD-API-KEY:)([^&]+)/, '\1[REDACTED]')
|
34
|
+
logger.filter(/(DD-APPLICATION-KEY:)([^&]+)/, '\1[REDACTED]')
|
35
|
+
end
|
36
|
+
faraday.response :raise_error
|
37
|
+
faraday.response :json
|
38
|
+
faraday.adapter Faraday.default_adapter
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def api_url
|
43
|
+
ENV.fetch('DD_SITE_URL', "https://api.datadoghq.com/")
|
13
44
|
end
|
14
45
|
|
15
46
|
def api_version
|
@@ -30,7 +61,7 @@ module DatadogBackup
|
|
30
61
|
current = except(get_by_id(id)).deep_sort.to_yaml
|
31
62
|
filesystem = except(load_from_file_by_id(id)).deep_sort.to_yaml
|
32
63
|
result = ::Diffy::Diff.new(current, filesystem, include_plus_and_minus_in_html: true).to_s(diff_format)
|
33
|
-
|
64
|
+
LOGGER.debug("Compared ID #{id} and found #{result}")
|
34
65
|
result
|
35
66
|
end
|
36
67
|
|
@@ -44,19 +75,17 @@ module DatadogBackup
|
|
44
75
|
end
|
45
76
|
|
46
77
|
def get(id)
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
return {} if e.message.include?('Request failed with error ["404"')
|
52
|
-
|
53
|
-
raise e.message
|
78
|
+
params = {}
|
79
|
+
headers = {}
|
80
|
+
response = api_service.get("/api/#{api_version}/#{api_resource_name}/#{id}", params, headers)
|
81
|
+
body_with_2xx(response)
|
54
82
|
end
|
55
83
|
|
56
84
|
def get_all
|
57
|
-
|
58
|
-
|
59
|
-
|
85
|
+
params = {}
|
86
|
+
headers = {}
|
87
|
+
response = api_service.get("/api/#{api_version}/#{api_resource_name}", params, headers)
|
88
|
+
body_with_2xx(response)
|
60
89
|
end
|
61
90
|
|
62
91
|
def get_and_write_file(id)
|
@@ -79,20 +108,20 @@ module DatadogBackup
|
|
79
108
|
|
80
109
|
# Calls out to Datadog and checks for a '200' response
|
81
110
|
def create(body)
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
111
|
+
headers = {}
|
112
|
+
response = api_service.post("/api/#{api_version}/#{api_resource_name}", body, headers)
|
113
|
+
body = body_with_2xx(response)
|
114
|
+
LOGGER.warn "Successfully created #{body.fetch('id')} in datadog."
|
115
|
+
body
|
87
116
|
end
|
88
117
|
|
89
118
|
# Calls out to Datadog and checks for a '200' response
|
90
119
|
def update(id, body)
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
120
|
+
headers = {}
|
121
|
+
response = api_service.put("/api/#{api_version}/#{api_resource_name}/#{id}", body, headers)
|
122
|
+
body = body_with_2xx(response)
|
123
|
+
LOGGER.warn 'Successfully restored #{id} to datadog.'
|
124
|
+
body
|
96
125
|
end
|
97
126
|
|
98
127
|
def restore(id)
|
@@ -100,7 +129,7 @@ module DatadogBackup
|
|
100
129
|
begin
|
101
130
|
update(id, body)
|
102
131
|
rescue RuntimeError => e
|
103
|
-
if e.message.include?('
|
132
|
+
if e.message.include?('update failed with error 404')
|
104
133
|
new_id = create(body).fetch('id')
|
105
134
|
|
106
135
|
FileUtils.rm(find_file_by_id(id))
|
@@ -111,21 +140,9 @@ module DatadogBackup
|
|
111
140
|
end
|
112
141
|
end
|
113
142
|
|
114
|
-
def
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
response = yield
|
119
|
-
raise "Request failed with error #{response}" unless response[0] == '200'
|
120
|
-
|
121
|
-
response[1]
|
122
|
-
rescue ::Net::OpenTimeout => e
|
123
|
-
if (retries += 1) <= max_retries
|
124
|
-
sleep(0.1 * retries**5) # 0.1, 3.2, 24.3, 102.4 seconds per retry
|
125
|
-
retry
|
126
|
-
else
|
127
|
-
raise "Net::OpenTimeout: #{e.message}"
|
128
|
-
end
|
143
|
+
def body_with_2xx(response)
|
144
|
+
raise "#{caller_locations(1,1)[0].label} failed with error #{response.status}" unless response.status.to_s =~ /^2/
|
145
|
+
response.body
|
129
146
|
end
|
130
147
|
end
|
131
148
|
end
|
@@ -2,14 +2,6 @@
|
|
2
2
|
|
3
3
|
module DatadogBackup
|
4
4
|
class Dashboards < Core
|
5
|
-
def all_boards
|
6
|
-
get_all.fetch('dashboards')
|
7
|
-
end
|
8
|
-
|
9
|
-
def api_service
|
10
|
-
# The underlying class from Dogapi that talks to datadog
|
11
|
-
client.instance_variable_get(:@dashboard_service)
|
12
|
-
end
|
13
5
|
|
14
6
|
def api_version
|
15
7
|
'v1'
|
@@ -20,21 +12,25 @@ module DatadogBackup
|
|
20
12
|
end
|
21
13
|
|
22
14
|
def backup
|
23
|
-
|
15
|
+
LOGGER.info("Starting diffs on #{::DatadogBackup::ThreadPool::TPOOL.max_length} threads")
|
24
16
|
|
25
|
-
futures =
|
17
|
+
futures = all_dashboards.map do |board|
|
26
18
|
Concurrent::Promises.future_on(::DatadogBackup::ThreadPool::TPOOL, board) do |board|
|
27
19
|
id = board['id']
|
28
20
|
get_and_write_file(id)
|
29
21
|
end
|
30
22
|
end
|
31
23
|
|
32
|
-
watcher = ::DatadogBackup::ThreadPool.watcher
|
24
|
+
watcher = ::DatadogBackup::ThreadPool.watcher
|
33
25
|
watcher.join if watcher.status
|
34
26
|
|
35
27
|
Concurrent::Promises.zip(*futures).value!
|
36
28
|
end
|
37
29
|
|
30
|
+
def all_dashboards
|
31
|
+
get_all.fetch('dashboards')
|
32
|
+
end
|
33
|
+
|
38
34
|
def initialize(options)
|
39
35
|
super(options)
|
40
36
|
@banlist = %w[modified_at url].freeze
|
@@ -10,30 +10,14 @@ module DatadogBackup
|
|
10
10
|
@options[:backup_dir]
|
11
11
|
end
|
12
12
|
|
13
|
-
def client
|
14
|
-
@options[:client]
|
15
|
-
end
|
16
|
-
|
17
13
|
def concurrency_limit
|
18
14
|
@options[:concurrency_limit] | 2
|
19
15
|
end
|
20
16
|
|
21
|
-
def datadog_api_key
|
22
|
-
@options[:datadog_api_key]
|
23
|
-
end
|
24
|
-
|
25
|
-
def datadog_app_key
|
26
|
-
@options[:datadog_app_key]
|
27
|
-
end
|
28
|
-
|
29
17
|
def diff_format
|
30
18
|
@options[:diff_format]
|
31
19
|
end
|
32
20
|
|
33
|
-
def logger
|
34
|
-
@options[:logger]
|
35
|
-
end
|
36
|
-
|
37
21
|
# Either :json or :yaml
|
38
22
|
def output_format
|
39
23
|
@options[:output_format]
|
@@ -8,21 +8,21 @@ module DatadogBackup
|
|
8
8
|
fallback_policy: :abort
|
9
9
|
)
|
10
10
|
|
11
|
-
def self.watcher
|
11
|
+
def self.watcher
|
12
12
|
Thread.new(TPOOL) do |pool|
|
13
13
|
while pool.queue_length.positive?
|
14
14
|
sleep 2
|
15
|
-
|
15
|
+
LOGGER.info("#{pool.queue_length} tasks remaining for execution.")
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
-
def self.shutdown
|
21
|
-
|
20
|
+
def self.shutdown
|
21
|
+
LOGGER.fatal 'Shutdown signal caught. Performing orderly shut down of thread pool. Press Ctrl+C again to forcibly shut down, but be warned, DATA LOSS MAY OCCUR.'
|
22
22
|
TPOOL.shutdown
|
23
23
|
TPOOL.wait_for_termination
|
24
24
|
rescue SystemExit, Interrupt
|
25
|
-
|
25
|
+
LOGGER.fatal 'OK Nuking, DATA LOSS MAY OCCUR.'
|
26
26
|
TPOOL.kill
|
27
27
|
end
|
28
28
|
end
|