datadog_backup 2.0.1 → 3.0.0.alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -0
- data/Gemfile.lock +23 -12
- data/README.md +21 -9
- data/bin/datadog_backup +10 -18
- data/datadog_backup.gemspec +9 -6
- data/example/.github/workflows/backup.yml +2 -2
- data/lib/datadog_backup/cli.rb +4 -13
- data/lib/datadog_backup/core.rb +55 -38
- data/lib/datadog_backup/dashboards.rb +7 -11
- data/lib/datadog_backup/deprecations.rb +11 -0
- data/lib/datadog_backup/local_filesystem.rb +1 -1
- data/lib/datadog_backup/monitors.rb +0 -5
- data/lib/datadog_backup/options.rb +0 -16
- data/lib/datadog_backup/thread_pool.rb +5 -5
- data/lib/datadog_backup/version.rb +1 -1
- data/lib/datadog_backup.rb +4 -3
- data/spec/datadog_backup/cli_spec.rb +13 -57
- data/spec/datadog_backup/core_spec.rb +29 -63
- data/spec/datadog_backup/dashboards_spec.rb +15 -16
- data/spec/datadog_backup/deprecations_spec.rb +36 -0
- data/spec/datadog_backup/local_filesystem_spec.rb +2 -7
- data/spec/datadog_backup/monitors_spec.rb +17 -16
- data/spec/datadog_backup_bin_spec.rb +5 -5
- data/spec/spec_helper.rb +7 -6
- metadata +57 -26
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ada6cc077ab1a7145bdbee5db768e9544a50a20f427c49bf86aabfe63ab2b74
|
4
|
+
data.tar.gz: 0e40617381ac33d1b4094cd277303dca2d0ece0372f7f13a3b4161eac304f34c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 437faae8390ce834b7a20bfcf6f2097f94a8ab93a808aa0c5594e1972e0e8b9497e16838dda7b02f89349b8a41c721e4577ea552cf1b267ea923b016d00f77dc
|
7
|
+
data.tar.gz: 9a96702e3928e1768adc888dc206cdf13e4a7a3adb9b99a385dd487338b52ff62028de96e87220eb487b9959d73e58a1a02d12f125c906b8e62209ad6e65d94d
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,32 @@
|
|
1
|
+
# [3.0.0-alpha.2](https://github.com/scribd/datadog_backup/compare/v3.0.0-alpha.1...v3.0.0-alpha.2) (2022-08-25)
|
2
|
+
|
3
|
+
|
4
|
+
### Bug Fixes
|
5
|
+
|
6
|
+
* remove development pry ([611d0a6](https://github.com/scribd/datadog_backup/commit/611d0a6dd5899b0046fc00233cf679834b275089))
|
7
|
+
|
8
|
+
# [3.0.0-alpha.1](https://github.com/scribd/datadog_backup/compare/v2.0.2...v3.0.0-alpha.1) (2022-08-24)
|
9
|
+
|
10
|
+
|
11
|
+
* feat!: release 3.0 ([d09d9e6](https://github.com/scribd/datadog_backup/commit/d09d9e6c845edb35c49cbb19ec6b35878304a078))
|
12
|
+
|
13
|
+
|
14
|
+
### BREAKING CHANGES
|
15
|
+
|
16
|
+
* DATADOG_API_KEY and DATADOG_APP_KEY are no longer the environment variables used to authenticate to Datadog. Instead, set the environment variables DD_API_KEY and DD_APP_KEY.
|
17
|
+
* ruby 2.6 is no longer supported. Please upgrade to ruby 2.7 or higher.
|
18
|
+
* The options `--ssh` and `--ssshh` are no longer supported. Instead, please use `--quiet` to supress logging. `--debug` remains supported.
|
19
|
+
* The environment variable `DATADOG_HOST` is no longer supported. Instead, please use `DD_SITE_URL`.
|
20
|
+
|
21
|
+
refactor: The legacy [dogapi-rb ](https://github.com/DataDog/dogapi-rb) gem is replaced with [faraday](https://lostisland.github.io/faraday/). The [official client library](https://github.com/DataDog/datadog-api-client-ruby) was considered, but was not adopted as I had a hard time grok-ing it.
|
22
|
+
|
23
|
+
## [2.0.2](https://github.com/scribd/datadog_backup/compare/v2.0.1...v2.0.2) (2022-08-11)
|
24
|
+
|
25
|
+
|
26
|
+
### Bug Fixes
|
27
|
+
|
28
|
+
* Deprecate Ruby 2.6 and Drop support for Ruby 2.5 ([#132](https://github.com/scribd/datadog_backup/issues/132)) ([432cb2c](https://github.com/scribd/datadog_backup/commit/432cb2c0d8b12d89aef81cf35597aa90f77407eb))
|
29
|
+
|
1
30
|
## [2.0.1](https://github.com/scribd/datadog_backup/compare/v2.0.0...v2.0.1) (2022-08-11)
|
2
31
|
|
3
32
|
|
data/Gemfile.lock
CHANGED
@@ -1,25 +1,31 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
datadog_backup (0.
|
5
|
-
amazing_print
|
6
|
-
concurrent-ruby
|
7
|
-
deepsort
|
8
|
-
diffy
|
9
|
-
|
4
|
+
datadog_backup (3.0.0.alpha.1)
|
5
|
+
amazing_print
|
6
|
+
concurrent-ruby
|
7
|
+
deepsort
|
8
|
+
diffy
|
9
|
+
faraday
|
10
|
+
faraday-retry
|
10
11
|
|
11
12
|
GEM
|
12
13
|
remote: https://rubygems.org/
|
13
14
|
specs:
|
14
15
|
amazing_print (1.4.0)
|
15
16
|
ast (2.4.2)
|
17
|
+
byebug (11.1.3)
|
16
18
|
coderay (1.1.3)
|
17
19
|
concurrent-ruby (1.1.10)
|
18
20
|
deepsort (0.4.5)
|
19
21
|
diff-lcs (1.5.0)
|
20
22
|
diffy (3.4.2)
|
21
|
-
|
22
|
-
|
23
|
+
faraday (2.5.2)
|
24
|
+
faraday-net_http (>= 2.0, < 3.1)
|
25
|
+
ruby2_keywords (>= 0.0.4)
|
26
|
+
faraday-net_http (3.0.0)
|
27
|
+
faraday-retry (2.0.0)
|
28
|
+
faraday (~> 2.0)
|
23
29
|
ffi (1.15.5)
|
24
30
|
formatador (1.1.0)
|
25
31
|
guard (2.18.0)
|
@@ -42,7 +48,6 @@ GEM
|
|
42
48
|
rb-inotify (~> 0.9, >= 0.9.10)
|
43
49
|
lumberjack (1.2.8)
|
44
50
|
method_source (1.0.0)
|
45
|
-
multi_json (1.15.0)
|
46
51
|
nenv (0.3.0)
|
47
52
|
notiffany (0.1.3)
|
48
53
|
nenv (~> 0.1)
|
@@ -53,6 +58,9 @@ GEM
|
|
53
58
|
pry (0.14.1)
|
54
59
|
coderay (~> 1.1)
|
55
60
|
method_source (~> 1.0)
|
61
|
+
pry-byebug (3.10.1)
|
62
|
+
byebug (~> 11.0)
|
63
|
+
pry (>= 0.13, < 0.15)
|
56
64
|
rainbow (3.1.1)
|
57
65
|
rb-fsevent (0.11.1)
|
58
66
|
rb-inotify (0.10.1)
|
@@ -72,14 +80,14 @@ GEM
|
|
72
80
|
diff-lcs (>= 1.2.0, < 2.0)
|
73
81
|
rspec-support (~> 3.11.0)
|
74
82
|
rspec-support (3.11.0)
|
75
|
-
rubocop (1.
|
83
|
+
rubocop (1.35.1)
|
76
84
|
json (~> 2.3)
|
77
85
|
parallel (~> 1.10)
|
78
86
|
parser (>= 3.1.2.1)
|
79
87
|
rainbow (>= 2.2.2, < 4.0)
|
80
88
|
regexp_parser (>= 1.8, < 3.0)
|
81
89
|
rexml (>= 3.2.5, < 4.0)
|
82
|
-
rubocop-ast (>= 1.20.
|
90
|
+
rubocop-ast (>= 1.20.1, < 2.0)
|
83
91
|
ruby-progressbar (~> 1.7)
|
84
92
|
unicode-display_width (>= 1.4.0, < 3.0)
|
85
93
|
rubocop-ast (1.21.0)
|
@@ -87,21 +95,24 @@ GEM
|
|
87
95
|
rubocop-rspec (2.12.1)
|
88
96
|
rubocop (~> 1.31)
|
89
97
|
ruby-progressbar (1.11.0)
|
98
|
+
ruby2_keywords (0.0.5)
|
90
99
|
shellany (0.0.1)
|
91
100
|
thor (1.2.1)
|
92
101
|
unicode-display_width (2.2.0)
|
93
102
|
|
94
103
|
PLATFORMS
|
95
104
|
ruby
|
105
|
+
x86_64-linux
|
96
106
|
|
97
107
|
DEPENDENCIES
|
98
108
|
bundler
|
99
109
|
datadog_backup!
|
100
110
|
guard-rspec
|
101
111
|
pry
|
112
|
+
pry-byebug
|
102
113
|
rspec
|
103
114
|
rubocop
|
104
115
|
rubocop-rspec
|
105
116
|
|
106
117
|
BUNDLED WITH
|
107
|
-
2.3.
|
118
|
+
2.3.20
|
data/README.md
CHANGED
@@ -11,6 +11,20 @@ Currently supports
|
|
11
11
|
|
12
12
|
Additional features may be built out over time.
|
13
13
|
|
14
|
+
# v3 Migration
|
15
|
+
|
16
|
+
## Breaking Changes
|
17
|
+
v3 is a backwards incompatible change.
|
18
|
+
|
19
|
+
- [] DATADOG_API_KEY and DATADOG_APP_KEY are no longer the environment variables used to authenticate to Datadog. Instead, set the environment variables DD_API_KEY and DD_APP_KEY.
|
20
|
+
- [ ] ruby 2.6 is no longer supported. Please upgrade to ruby 2.7 or higher.
|
21
|
+
- [ ] The options `--ssh` and `--ssshh` are no longer supported. Instead, please use `--quiet` to supress logging. `--debug` remains supported.
|
22
|
+
- [ ] The environment variable `DATADOG_HOST` is no longer supported. Instead, please use `DD_SITE_URL`.
|
23
|
+
|
24
|
+
## Misc
|
25
|
+
- [ ] The legacy [dogapi-rb ](https://github.com/DataDog/dogapi-rb) gem is replaced with [faraday](https://lostisland.github.io/faraday/). The [official client library](https://github.com/DataDog/datadog-api-client-ruby) was considered, but was not adopted as I had a hard time grok-ing it.
|
26
|
+
|
27
|
+
|
14
28
|
## Installation
|
15
29
|
|
16
30
|
```
|
@@ -22,13 +36,13 @@ gem install datadog_backup
|
|
22
36
|

|
23
37
|
|
24
38
|
```
|
25
|
-
|
39
|
+
DD_API_KEY=example123 DD_APP_KEY=example123 datadog_backup <backup|diffs|restore> [--backup-dir /path/to/backups] [--debug] [--monitors-only] [--dashboards-only] [--diff-format color|html|html_simple] [--no-color] [--json]
|
26
40
|
```
|
27
41
|
|
28
42
|
```
|
29
43
|
gem install datadog_backup
|
30
|
-
export
|
31
|
-
export
|
44
|
+
export DD_API_KEY=abc123
|
45
|
+
export DD_APP_KEY=abc123
|
32
46
|
|
33
47
|
# Perform backup to `./backup/` using YAML encoding
|
34
48
|
datadog_backup backup
|
@@ -49,8 +63,7 @@ Supply the following parameters in order to customize datadog_backup:
|
|
49
63
|
parameter | description | default
|
50
64
|
---------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------
|
51
65
|
--debug | log debug and above | info
|
52
|
-
--
|
53
|
-
--shhh | log errors and above | info
|
66
|
+
--quiet | only show errors and above | info
|
54
67
|
--backup-dir PATH | path to the directory to backup to or restore from | `./backup/`
|
55
68
|
--monitors-only | only backup monitors | backup monitors and dashboards
|
56
69
|
--dashboards-only | only backup dashboards | backup monitors and dashboards
|
@@ -66,10 +79,9 @@ The following environment variables can be set in order to further customize dat
|
|
66
79
|
|
67
80
|
environment variable | description | default
|
68
81
|
---------------------|--------------------------------------------------------------------------------|--------------------------
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
dd_proxy_https | Same as `http_proxy` | none
|
82
|
+
DD_SITE_URL | Describe the API endpoint to connect to (https://api.datadoghq.eu for example) | https://api.datadoghq.com
|
83
|
+
DD_API_KEY | The API key for the Datadog account | none
|
84
|
+
DD_API_KEY | The Application key for the Datadog account | none
|
73
85
|
|
74
86
|
|
75
87
|
### Usage in a Github repo
|
data/bin/datadog_backup
CHANGED
@@ -5,23 +5,21 @@ $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '../lib')
|
|
5
5
|
|
6
6
|
require 'logger'
|
7
7
|
require 'optparse'
|
8
|
-
|
9
|
-
require 'datadog_backup'
|
10
|
-
require 'dogapi'
|
11
|
-
|
12
8
|
$stdout.sync = $stderr.sync = true
|
13
9
|
LOGGER = Logger.new($stderr) unless defined?(LOGGER)
|
14
10
|
LOGGER.level = Logger::INFO
|
15
11
|
|
12
|
+
require 'datadog_backup'
|
13
|
+
|
14
|
+
|
16
15
|
def fatal(message)
|
17
16
|
LOGGER.fatal(message)
|
18
17
|
exit 1
|
19
18
|
end
|
20
19
|
|
21
20
|
def options_valid?(options)
|
22
|
-
%w[backup diffs restore].include?(options[:action])
|
23
|
-
|
24
|
-
options[:datadog_app_key]
|
21
|
+
%w[backup diffs restore].include?(options[:action])
|
22
|
+
%w[DD_API_KEY DD_APP_KEY].all? { |key| ENV[key] }
|
25
23
|
end
|
26
24
|
|
27
25
|
def prereqs(defaults)
|
@@ -30,7 +28,7 @@ def prereqs(defaults)
|
|
30
28
|
result = defaults.dup
|
31
29
|
|
32
30
|
options = OptionParser.new do |opts|
|
33
|
-
opts.banner = "Usage:
|
31
|
+
opts.banner = "Usage: DD_API_KEY=abc123 DD_APP_KEY=abc123 #{File.basename($PROGRAM_NAME)} <backup|diffs|restore>"
|
34
32
|
opts.separator ''
|
35
33
|
opts.on_tail('-h', '--help', 'Show this message') do
|
36
34
|
puts opts
|
@@ -39,10 +37,7 @@ def prereqs(defaults)
|
|
39
37
|
opts.on('--debug', 'log debug and above') do
|
40
38
|
LOGGER.level = Logger::DEBUG
|
41
39
|
end
|
42
|
-
opts.on('--
|
43
|
-
LOGGER.level = Logger::WARN
|
44
|
-
end
|
45
|
-
opts.on('--shhh', 'log errors and above') do
|
40
|
+
opts.on('--quiet', 'log errors and above') do
|
46
41
|
LOGGER.level = Logger::ERROR
|
47
42
|
end
|
48
43
|
opts.on('--backup-dir PATH', '`backup` by default') do |path|
|
@@ -73,7 +68,7 @@ def prereqs(defaults)
|
|
73
68
|
options.parse!
|
74
69
|
|
75
70
|
result[:action] = ARGV.first
|
76
|
-
fatal(options) unless options_valid?(result)
|
71
|
+
fatal(options.banner) unless options_valid?(result)
|
77
72
|
result
|
78
73
|
end
|
79
74
|
|
@@ -81,14 +76,11 @@ end
|
|
81
76
|
# Default parameters
|
82
77
|
defaults = {
|
83
78
|
action: nil,
|
84
|
-
datadog_api_key: ENV.fetch('DATADOG_API_KEY', nil),
|
85
|
-
datadog_app_key: ENV.fetch('DATADOG_APP_KEY', nil),
|
86
79
|
backup_dir: File.join(ENV.fetch('PWD'), 'backup'),
|
87
80
|
diff_format: :color,
|
88
81
|
resources: [DatadogBackup::Dashboards, DatadogBackup::Monitors],
|
89
82
|
output_format: :yaml,
|
90
|
-
force_restore: false
|
91
|
-
logger: LOGGER
|
83
|
+
force_restore: false
|
92
84
|
}
|
93
85
|
|
94
|
-
DatadogBackup::Cli.new(prereqs(defaults)).run!
|
86
|
+
DatadogBackup::Cli.new(prereqs(defaults)).run!
|
data/datadog_backup.gemspec
CHANGED
@@ -19,16 +19,19 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.test_files = spec.files.grep(%r{^spec/})
|
20
20
|
spec.require_paths = ['lib']
|
21
21
|
|
22
|
-
spec.required_ruby_version = ['>= 2.
|
22
|
+
spec.required_ruby_version = ['>= 2.7']
|
23
|
+
|
24
|
+
spec.add_dependency 'amazing_print'
|
25
|
+
spec.add_dependency 'concurrent-ruby'
|
26
|
+
spec.add_dependency 'deepsort'
|
27
|
+
spec.add_dependency 'diffy'
|
28
|
+
spec.add_dependency 'faraday'
|
29
|
+
spec.add_dependency 'faraday-retry'
|
23
30
|
|
24
|
-
spec.add_dependency 'amazing_print', '1.4.0'
|
25
|
-
spec.add_dependency 'concurrent-ruby', '1.1.10'
|
26
|
-
spec.add_dependency 'deepsort', '0.4.5'
|
27
|
-
spec.add_dependency 'diffy', '3.4.2'
|
28
|
-
spec.add_dependency 'dogapi', '1.45.0'
|
29
31
|
|
30
32
|
spec.add_development_dependency 'bundler'
|
31
33
|
spec.add_development_dependency 'pry'
|
34
|
+
spec.add_development_dependency 'pry-byebug'
|
32
35
|
spec.add_development_dependency 'guard-rspec'
|
33
36
|
spec.add_development_dependency 'rspec'
|
34
37
|
spec.add_development_dependency 'rubocop'
|
@@ -17,8 +17,8 @@ jobs:
|
|
17
17
|
ruby-version: 2.7.1
|
18
18
|
- name: perform backup
|
19
19
|
env:
|
20
|
-
|
21
|
-
|
20
|
+
DD_API_KEY: ${{ secrets.DD_API_KEY }}
|
21
|
+
DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
|
22
22
|
run: |
|
23
23
|
gem install --no-document bundler
|
24
24
|
bundle install --jobs 4 --retry 3
|
data/lib/datadog_backup/cli.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'optparse'
|
4
|
-
require 'logger'
|
5
4
|
require 'amazing_print'
|
6
5
|
|
7
6
|
module DatadogBackup
|
@@ -9,7 +8,7 @@ module DatadogBackup
|
|
9
8
|
include ::DatadogBackup::Options
|
10
9
|
|
11
10
|
def all_diff_futures
|
12
|
-
|
11
|
+
LOGGER.info("Starting diffs on #{::DatadogBackup::ThreadPool::TPOOL.max_length} threads")
|
13
12
|
any_resource_instance
|
14
13
|
.all_file_ids_for_selected_resources
|
15
14
|
.map do |id|
|
@@ -29,20 +28,13 @@ module DatadogBackup
|
|
29
28
|
any_resource_instance.all_files
|
30
29
|
end
|
31
30
|
|
32
|
-
def initialize_client
|
33
|
-
@options[:client] ||= Dogapi::Client.new(
|
34
|
-
datadog_api_key,
|
35
|
-
datadog_app_key
|
36
|
-
)
|
37
|
-
end
|
38
|
-
|
39
31
|
def definitive_resource_instance(id)
|
40
32
|
matching_resource_instance(any_resource_instance.class_from_id(id))
|
41
33
|
end
|
42
34
|
|
43
35
|
def diffs
|
44
36
|
futures = all_diff_futures
|
45
|
-
::DatadogBackup::ThreadPool.watcher
|
37
|
+
::DatadogBackup::ThreadPool.watcher.join
|
46
38
|
|
47
39
|
format_diff_output(
|
48
40
|
Concurrent::Promises
|
@@ -87,7 +79,6 @@ module DatadogBackup
|
|
87
79
|
|
88
80
|
def initialize(options)
|
89
81
|
@options = options
|
90
|
-
initialize_client
|
91
82
|
end
|
92
83
|
|
93
84
|
def matching_resource_instance(klass)
|
@@ -102,7 +93,7 @@ module DatadogBackup
|
|
102
93
|
|
103
94
|
def restore
|
104
95
|
futures = all_diff_futures
|
105
|
-
watcher = ::DatadogBackup::ThreadPool.watcher
|
96
|
+
watcher = ::DatadogBackup::ThreadPool.watcher
|
106
97
|
|
107
98
|
futures.each do |future|
|
108
99
|
id, diff = *future.value!
|
@@ -138,7 +129,7 @@ module DatadogBackup
|
|
138
129
|
def run!
|
139
130
|
puts(send(action.to_sym))
|
140
131
|
rescue SystemExit, Interrupt
|
141
|
-
::DatadogBackup::ThreadPool.shutdown
|
132
|
+
::DatadogBackup::ThreadPool.shutdown
|
142
133
|
end
|
143
134
|
end
|
144
135
|
end
|
data/lib/datadog_backup/core.rb
CHANGED
@@ -2,14 +2,45 @@
|
|
2
2
|
|
3
3
|
require 'diffy'
|
4
4
|
require 'deepsort'
|
5
|
+
require 'faraday'
|
6
|
+
require 'faraday/retry'
|
7
|
+
|
8
|
+
|
5
9
|
|
6
10
|
module DatadogBackup
|
7
11
|
class Core
|
8
12
|
include ::DatadogBackup::LocalFilesystem
|
9
13
|
include ::DatadogBackup::Options
|
10
14
|
|
15
|
+
@@retry_options = {
|
16
|
+
max: 5,
|
17
|
+
interval: 0.05,
|
18
|
+
interval_randomness: 0.5,
|
19
|
+
backoff_factor: 2
|
20
|
+
}
|
21
|
+
|
11
22
|
def api_service
|
12
|
-
|
23
|
+
conn ||= Faraday.new(
|
24
|
+
url: api_url,
|
25
|
+
headers: {
|
26
|
+
'DD-API-KEY' => ENV.fetch('DD_API_KEY'),
|
27
|
+
'DD-APPLICATION-KEY' => ENV.fetch('DD_APP_KEY')
|
28
|
+
}
|
29
|
+
) do |faraday|
|
30
|
+
faraday.request :json
|
31
|
+
faraday.request :retry, @@retry_options
|
32
|
+
faraday.response(:logger, LOGGER, {headers: true, bodies: LOGGER.debug?, log_level: :debug}) do | logger |
|
33
|
+
logger.filter(/(DD-API-KEY:)([^&]+)/, '\1[REDACTED]')
|
34
|
+
logger.filter(/(DD-APPLICATION-KEY:)([^&]+)/, '\1[REDACTED]')
|
35
|
+
end
|
36
|
+
faraday.response :raise_error
|
37
|
+
faraday.response :json
|
38
|
+
faraday.adapter Faraday.default_adapter
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def api_url
|
43
|
+
ENV.fetch('DD_SITE_URL', "https://api.datadoghq.com/")
|
13
44
|
end
|
14
45
|
|
15
46
|
def api_version
|
@@ -30,7 +61,7 @@ module DatadogBackup
|
|
30
61
|
current = except(get_by_id(id)).deep_sort.to_yaml
|
31
62
|
filesystem = except(load_from_file_by_id(id)).deep_sort.to_yaml
|
32
63
|
result = ::Diffy::Diff.new(current, filesystem, include_plus_and_minus_in_html: true).to_s(diff_format)
|
33
|
-
|
64
|
+
LOGGER.debug("Compared ID #{id} and found #{result}")
|
34
65
|
result
|
35
66
|
end
|
36
67
|
|
@@ -44,19 +75,17 @@ module DatadogBackup
|
|
44
75
|
end
|
45
76
|
|
46
77
|
def get(id)
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
return {} if e.message.include?('Request failed with error ["404"')
|
52
|
-
|
53
|
-
raise e.message
|
78
|
+
params = {}
|
79
|
+
headers = {}
|
80
|
+
response = api_service.get("/api/#{api_version}/#{api_resource_name}/#{id}", params, headers)
|
81
|
+
body_with_2xx(response)
|
54
82
|
end
|
55
83
|
|
56
84
|
def get_all
|
57
|
-
|
58
|
-
|
59
|
-
|
85
|
+
params = {}
|
86
|
+
headers = {}
|
87
|
+
response = api_service.get("/api/#{api_version}/#{api_resource_name}", params, headers)
|
88
|
+
body_with_2xx(response)
|
60
89
|
end
|
61
90
|
|
62
91
|
def get_and_write_file(id)
|
@@ -79,20 +108,20 @@ module DatadogBackup
|
|
79
108
|
|
80
109
|
# Calls out to Datadog and checks for a '200' response
|
81
110
|
def create(body)
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
111
|
+
headers = {}
|
112
|
+
response = api_service.post("/api/#{api_version}/#{api_resource_name}", body, headers)
|
113
|
+
body = body_with_2xx(response)
|
114
|
+
LOGGER.warn "Successfully created #{body.fetch('id')} in datadog."
|
115
|
+
body
|
87
116
|
end
|
88
117
|
|
89
118
|
# Calls out to Datadog and checks for a '200' response
|
90
119
|
def update(id, body)
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
120
|
+
headers = {}
|
121
|
+
response = api_service.put("/api/#{api_version}/#{api_resource_name}/#{id}", body, headers)
|
122
|
+
body = body_with_2xx(response)
|
123
|
+
LOGGER.warn 'Successfully restored #{id} to datadog.'
|
124
|
+
body
|
96
125
|
end
|
97
126
|
|
98
127
|
def restore(id)
|
@@ -100,7 +129,7 @@ module DatadogBackup
|
|
100
129
|
begin
|
101
130
|
update(id, body)
|
102
131
|
rescue RuntimeError => e
|
103
|
-
if e.message.include?('
|
132
|
+
if e.message.include?('update failed with error 404')
|
104
133
|
new_id = create(body).fetch('id')
|
105
134
|
|
106
135
|
FileUtils.rm(find_file_by_id(id))
|
@@ -111,21 +140,9 @@ module DatadogBackup
|
|
111
140
|
end
|
112
141
|
end
|
113
142
|
|
114
|
-
def
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
response = yield
|
119
|
-
raise "Request failed with error #{response}" unless response[0] == '200'
|
120
|
-
|
121
|
-
response[1]
|
122
|
-
rescue ::Net::OpenTimeout => e
|
123
|
-
if (retries += 1) <= max_retries
|
124
|
-
sleep(0.1 * retries**5) # 0.1, 3.2, 24.3, 102.4 seconds per retry
|
125
|
-
retry
|
126
|
-
else
|
127
|
-
raise "Net::OpenTimeout: #{e.message}"
|
128
|
-
end
|
143
|
+
def body_with_2xx(response)
|
144
|
+
raise "#{caller_locations(1,1)[0].label} failed with error #{response.status}" unless response.status.to_s =~ /^2/
|
145
|
+
response.body
|
129
146
|
end
|
130
147
|
end
|
131
148
|
end
|
@@ -2,14 +2,6 @@
|
|
2
2
|
|
3
3
|
module DatadogBackup
|
4
4
|
class Dashboards < Core
|
5
|
-
def all_boards
|
6
|
-
get_all.fetch('dashboards')
|
7
|
-
end
|
8
|
-
|
9
|
-
def api_service
|
10
|
-
# The underlying class from Dogapi that talks to datadog
|
11
|
-
client.instance_variable_get(:@dashboard_service)
|
12
|
-
end
|
13
5
|
|
14
6
|
def api_version
|
15
7
|
'v1'
|
@@ -20,21 +12,25 @@ module DatadogBackup
|
|
20
12
|
end
|
21
13
|
|
22
14
|
def backup
|
23
|
-
|
15
|
+
LOGGER.info("Starting diffs on #{::DatadogBackup::ThreadPool::TPOOL.max_length} threads")
|
24
16
|
|
25
|
-
futures =
|
17
|
+
futures = all_dashboards.map do |board|
|
26
18
|
Concurrent::Promises.future_on(::DatadogBackup::ThreadPool::TPOOL, board) do |board|
|
27
19
|
id = board['id']
|
28
20
|
get_and_write_file(id)
|
29
21
|
end
|
30
22
|
end
|
31
23
|
|
32
|
-
watcher = ::DatadogBackup::ThreadPool.watcher
|
24
|
+
watcher = ::DatadogBackup::ThreadPool.watcher
|
33
25
|
watcher.join if watcher.status
|
34
26
|
|
35
27
|
Concurrent::Promises.zip(*futures).value!
|
36
28
|
end
|
37
29
|
|
30
|
+
def all_dashboards
|
31
|
+
get_all.fetch('dashboards')
|
32
|
+
end
|
33
|
+
|
38
34
|
def initialize(options)
|
39
35
|
super(options)
|
40
36
|
@banlist = %w[modified_at url].freeze
|
@@ -10,30 +10,14 @@ module DatadogBackup
|
|
10
10
|
@options[:backup_dir]
|
11
11
|
end
|
12
12
|
|
13
|
-
def client
|
14
|
-
@options[:client]
|
15
|
-
end
|
16
|
-
|
17
13
|
def concurrency_limit
|
18
14
|
@options[:concurrency_limit] | 2
|
19
15
|
end
|
20
16
|
|
21
|
-
def datadog_api_key
|
22
|
-
@options[:datadog_api_key]
|
23
|
-
end
|
24
|
-
|
25
|
-
def datadog_app_key
|
26
|
-
@options[:datadog_app_key]
|
27
|
-
end
|
28
|
-
|
29
17
|
def diff_format
|
30
18
|
@options[:diff_format]
|
31
19
|
end
|
32
20
|
|
33
|
-
def logger
|
34
|
-
@options[:logger]
|
35
|
-
end
|
36
|
-
|
37
21
|
# Either :json or :yaml
|
38
22
|
def output_format
|
39
23
|
@options[:output_format]
|
@@ -8,21 +8,21 @@ module DatadogBackup
|
|
8
8
|
fallback_policy: :abort
|
9
9
|
)
|
10
10
|
|
11
|
-
def self.watcher
|
11
|
+
def self.watcher
|
12
12
|
Thread.new(TPOOL) do |pool|
|
13
13
|
while pool.queue_length.positive?
|
14
14
|
sleep 2
|
15
|
-
|
15
|
+
LOGGER.info("#{pool.queue_length} tasks remaining for execution.")
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
-
def self.shutdown
|
21
|
-
|
20
|
+
def self.shutdown
|
21
|
+
LOGGER.fatal 'Shutdown signal caught. Performing orderly shut down of thread pool. Press Ctrl+C again to forcibly shut down, but be warned, DATA LOSS MAY OCCUR.'
|
22
22
|
TPOOL.shutdown
|
23
23
|
TPOOL.wait_for_termination
|
24
24
|
rescue SystemExit, Interrupt
|
25
|
-
|
25
|
+
LOGGER.fatal 'OK Nuking, DATA LOSS MAY OCCUR.'
|
26
26
|
TPOOL.kill
|
27
27
|
end
|
28
28
|
end
|