datadog_backup 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/rspec_and_release.yml +54 -0
- data/.gitignore +119 -0
- data/CHANGELOG.md +80 -0
- data/CODE_OF_CONDUCT.md +76 -0
- data/Gemfile +12 -0
- data/Guardfile +44 -0
- data/LICENSE +21 -0
- data/README.md +57 -0
- data/bin/datadog_backup +73 -0
- data/datadog_backup.gemspec +31 -0
- data/example/.github/workflows/backup.yml +31 -0
- data/example/.gitignore +1 -0
- data/example/Gemfile +3 -0
- data/example/README.md +17 -0
- data/lib/datadog_backup.rb +19 -0
- data/lib/datadog_backup/cli.rb +141 -0
- data/lib/datadog_backup/core.rb +113 -0
- data/lib/datadog_backup/dashboards.rb +50 -0
- data/lib/datadog_backup/local_filesystem.rb +86 -0
- data/lib/datadog_backup/monitors.rb +42 -0
- data/lib/datadog_backup/options.rb +46 -0
- data/lib/datadog_backup/thread_pool.rb +30 -0
- data/lib/datadog_backup/version.rb +5 -0
- data/release.config.js +43 -0
- data/spec/datadog_backup/cli_spec.rb +117 -0
- data/spec/datadog_backup/core_spec.rb +107 -0
- data/spec/datadog_backup/dashboards_spec.rb +102 -0
- data/spec/datadog_backup/local_filesystem_spec.rb +173 -0
- data/spec/datadog_backup/monitors_spec.rb +119 -0
- data/spec/datadog_backup_bin_spec.rb +59 -0
- data/spec/datadog_backup_spec.rb +6 -0
- data/spec/spec_helper.rb +37 -0
- metadata +199 -0
@@ -0,0 +1,86 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'fileutils'
|
4
|
+
require 'json'
|
5
|
+
require 'yaml'
|
6
|
+
require 'deepsort'
|
7
|
+
|
8
|
+
module DatadogBackup
|
9
|
+
module LocalFilesystem
|
10
|
+
##
|
11
|
+
# Meant to be mixed into DatadogBackup::Core
|
12
|
+
# Relies on @options[:backup_dir] and @options[:output_format]
|
13
|
+
|
14
|
+
def all_files
|
15
|
+
::Dir.glob(::File.join(backup_dir, '**', '*')).select { |f| ::File.file?(f) }
|
16
|
+
end
|
17
|
+
|
18
|
+
def all_file_ids
|
19
|
+
all_files.map { |file| ::File.basename(file, '.*') }
|
20
|
+
end
|
21
|
+
|
22
|
+
def all_file_ids_for_selected_resources
|
23
|
+
all_file_ids.select do |id|
|
24
|
+
resources.include? class_from_id(id)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def class_from_id(id)
|
29
|
+
class_string = ::File.dirname(find_file_by_id(id)).split('/').last.capitalize
|
30
|
+
::DatadogBackup.const_get(class_string)
|
31
|
+
end
|
32
|
+
|
33
|
+
def dump(object)
|
34
|
+
if output_format == :json
|
35
|
+
JSON.pretty_generate(object.deep_sort)
|
36
|
+
elsif output_format == :yaml
|
37
|
+
YAML.dump(object.deep_sort)
|
38
|
+
else
|
39
|
+
raise 'invalid output_format specified or not specified'
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def filename(id)
|
44
|
+
::File.join(mydir, "#{id}.#{output_format}")
|
45
|
+
end
|
46
|
+
|
47
|
+
def file_type(filepath)
|
48
|
+
::File.extname(filepath).strip.downcase[1..-1].to_sym
|
49
|
+
end
|
50
|
+
|
51
|
+
def find_file_by_id(id)
|
52
|
+
::Dir.glob(::File.join(backup_dir, '**', "#{id}.*")).first
|
53
|
+
end
|
54
|
+
|
55
|
+
def load_from_file(string, output_format)
|
56
|
+
if output_format == :json
|
57
|
+
JSON.parse(string)
|
58
|
+
elsif output_format == :yaml
|
59
|
+
YAML.safe_load(string)
|
60
|
+
else
|
61
|
+
raise 'invalid output_format specified or not specified'
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def load_from_file_by_id(id)
|
66
|
+
filepath = find_file_by_id(id)
|
67
|
+
load_from_file(::File.read(filepath), file_type(filepath))
|
68
|
+
end
|
69
|
+
|
70
|
+
def mydir
|
71
|
+
::File.join(backup_dir, myclass)
|
72
|
+
end
|
73
|
+
|
74
|
+
def purge
|
75
|
+
::FileUtils.rm(::Dir.glob(File.join(mydir, '*')))
|
76
|
+
end
|
77
|
+
|
78
|
+
def write_file(data, filename)
|
79
|
+
logger.info "Backing up #{filename}"
|
80
|
+
file = ::File.open(filename, 'w')
|
81
|
+
file.write(data)
|
82
|
+
ensure
|
83
|
+
file.close
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DatadogBackup
|
4
|
+
class Monitors < Core
|
5
|
+
API_SERVICE_NAME = :@monitor_svc
|
6
|
+
BANLIST = %w[overall_state overall_state_modified matching_downtimes modified]
|
7
|
+
|
8
|
+
def all_monitors
|
9
|
+
@all_monitors ||= client_with_200(:get_all_monitors)
|
10
|
+
end
|
11
|
+
|
12
|
+
def api_service
|
13
|
+
# The underlying class from Dogapi that talks to datadog
|
14
|
+
client.instance_variable_get(:@monitor_svc)
|
15
|
+
end
|
16
|
+
|
17
|
+
def api_version
|
18
|
+
'v1'
|
19
|
+
end
|
20
|
+
|
21
|
+
def api_resource_name
|
22
|
+
'monitor'
|
23
|
+
end
|
24
|
+
|
25
|
+
def backup
|
26
|
+
all_monitors.map do |monitor|
|
27
|
+
id = monitor['id']
|
28
|
+
write_file(dump(get_by_id(id)), filename(id))
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def diff(id)
|
33
|
+
super(id, BANLIST)
|
34
|
+
end
|
35
|
+
|
36
|
+
def get_by_id(id)
|
37
|
+
except(all_monitors.select { |monitor| monitor['id'].to_s == id.to_s }.first, BANLIST)
|
38
|
+
end
|
39
|
+
|
40
|
+
def restore!; end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DatadogBackup
|
4
|
+
module Options
|
5
|
+
def action
|
6
|
+
@options[:action]
|
7
|
+
end
|
8
|
+
|
9
|
+
def backup_dir
|
10
|
+
@options[:backup_dir]
|
11
|
+
end
|
12
|
+
|
13
|
+
def client
|
14
|
+
@options[:client]
|
15
|
+
end
|
16
|
+
|
17
|
+
def concurrency_limit
|
18
|
+
@options[:concurrency_limit] | 2
|
19
|
+
end
|
20
|
+
|
21
|
+
def datadog_api_key
|
22
|
+
@options[:datadog_api_key]
|
23
|
+
end
|
24
|
+
|
25
|
+
def datadog_app_key
|
26
|
+
@options[:datadog_app_key]
|
27
|
+
end
|
28
|
+
|
29
|
+
def diff_format
|
30
|
+
@options[:diff_format]
|
31
|
+
end
|
32
|
+
|
33
|
+
def logger
|
34
|
+
@options[:logger]
|
35
|
+
end
|
36
|
+
|
37
|
+
# Either :json or :yaml
|
38
|
+
def output_format
|
39
|
+
@options[:output_format]
|
40
|
+
end
|
41
|
+
|
42
|
+
def resources
|
43
|
+
@options[:resources]
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DatadogBackup
|
4
|
+
module ThreadPool
|
5
|
+
TPOOL = ::Concurrent::ThreadPoolExecutor.new(
|
6
|
+
min_threads: [2, Concurrent.processor_count].max,
|
7
|
+
max_threads: [2, Concurrent.processor_count].max * 2,
|
8
|
+
max_queue: [2, Concurrent.processor_count].max * 512,
|
9
|
+
fallback_policy: :abort
|
10
|
+
)
|
11
|
+
|
12
|
+
def self.watcher(logger)
|
13
|
+
Thread.new(TPOOL) do |pool|
|
14
|
+
while pool.queue_length > 0
|
15
|
+
sleep 2
|
16
|
+
logger.info("#{pool.queue_length} tasks remaining for execution.")
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.shutdown(logger)
|
22
|
+
logger.fatal 'Shutdown signal caught. Performing orderly shut down of thread pool. Press Ctrl+C again to forcibly shut down, but be warned, DATA LOSS MAY OCCUR.'
|
23
|
+
TPOOL.shutdown
|
24
|
+
TPOOL.wait_for_termination
|
25
|
+
rescue SystemExit, Interrupt
|
26
|
+
logger.fatal 'OK Nuking, DATA LOSS MAY OCCUR.'
|
27
|
+
TPOOL.kill
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/release.config.js
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
module.exports = {
|
2
|
+
"plugins": [
|
3
|
+
"@semantic-release/commit-analyzer",
|
4
|
+
"@semantic-release/release-notes-generator",
|
5
|
+
[
|
6
|
+
"@semantic-release/changelog",
|
7
|
+
{
|
8
|
+
"changelogFile": "CHANGELOG.md"
|
9
|
+
}
|
10
|
+
],
|
11
|
+
[
|
12
|
+
"semantic-release-rubygem",
|
13
|
+
{
|
14
|
+
"gemFileDir": "."
|
15
|
+
}
|
16
|
+
],
|
17
|
+
[
|
18
|
+
"@semantic-release/git",
|
19
|
+
{
|
20
|
+
"assets": [
|
21
|
+
"CHANGELOG.md"
|
22
|
+
],
|
23
|
+
"message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
|
24
|
+
}
|
25
|
+
],
|
26
|
+
[
|
27
|
+
"@semantic-release/github",
|
28
|
+
{
|
29
|
+
"assets": [
|
30
|
+
{
|
31
|
+
"path": "datadog_backup.zip",
|
32
|
+
"name": "datadog_backup.${nextRelease.version}.zip",
|
33
|
+
"label": "Full zip distribution"
|
34
|
+
},
|
35
|
+
{
|
36
|
+
"path": "datadog_backup-*.gem",
|
37
|
+
"label": "Gem distribution"
|
38
|
+
}
|
39
|
+
]
|
40
|
+
}
|
41
|
+
],
|
42
|
+
]
|
43
|
+
};
|
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
|
5
|
+
describe DatadogBackup::Cli do
|
6
|
+
let(:client_double) { double }
|
7
|
+
let(:tempdir) { Dir.mktmpdir }
|
8
|
+
let(:options) do
|
9
|
+
{
|
10
|
+
action: 'backup',
|
11
|
+
backup_dir: tempdir,
|
12
|
+
client: client_double,
|
13
|
+
datadog_api_key: 1,
|
14
|
+
datadog_app_key: 1,
|
15
|
+
diff_format: nil,
|
16
|
+
logger: Logger.new('/dev/null'),
|
17
|
+
output_format: :json,
|
18
|
+
resources: [DatadogBackup::Dashboards]
|
19
|
+
}
|
20
|
+
end
|
21
|
+
let(:cli) { DatadogBackup::Cli.new(options) }
|
22
|
+
let(:dashboards) { DatadogBackup::Dashboards.new(options) }
|
23
|
+
|
24
|
+
before(:example) do
|
25
|
+
allow(cli).to receive(:resource_instances).and_return([dashboards])
|
26
|
+
end
|
27
|
+
|
28
|
+
describe '#backup' do
|
29
|
+
context 'when dashboards are deleted in datadog' do
|
30
|
+
let(:all_boards) do
|
31
|
+
[
|
32
|
+
'200',
|
33
|
+
{
|
34
|
+
'dashboards' => [
|
35
|
+
{ 'id' => 'stillthere' },
|
36
|
+
{ 'id' => 'alsostillthere' }
|
37
|
+
]
|
38
|
+
}
|
39
|
+
]
|
40
|
+
end
|
41
|
+
before(:example) do
|
42
|
+
dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/stillthere.json")
|
43
|
+
dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/alsostillthere.json")
|
44
|
+
dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/deleted.json")
|
45
|
+
|
46
|
+
allow(client_double).to receive(:get_all_boards).and_return(all_boards)
|
47
|
+
allow(client_double).to receive(:get_board).and_return(['200', {}])
|
48
|
+
end
|
49
|
+
|
50
|
+
it 'deletes the file locally as well' do
|
51
|
+
cli.backup
|
52
|
+
expect { File.open("#{tempdir}/dashboards/deleted.json", 'r') }.to raise_error(Errno::ENOENT)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
describe '#diffs' do
|
58
|
+
before(:example) do
|
59
|
+
dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs1.json")
|
60
|
+
dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs2.json")
|
61
|
+
dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs3.json")
|
62
|
+
allow(dashboards).to receive(:get_by_id).and_return({ 'text' => 'diff2' })
|
63
|
+
allow(cli).to receive(:initialize_client).and_return(client_double)
|
64
|
+
end
|
65
|
+
subject { cli.diffs }
|
66
|
+
it {
|
67
|
+
is_expected.to include(
|
68
|
+
" ---\n id: diffs1\n ---\n-text: diff2\n+text: diff\n",
|
69
|
+
" ---\n id: diffs3\n ---\n-text: diff2\n+text: diff\n",
|
70
|
+
" ---\n id: diffs2\n ---\n-text: diff2\n+text: diff\n"
|
71
|
+
)
|
72
|
+
}
|
73
|
+
end
|
74
|
+
|
75
|
+
describe '#restore' do
|
76
|
+
before(:example) do
|
77
|
+
dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs1.json")
|
78
|
+
allow(dashboards).to receive(:get_by_id).and_return({ 'text' => 'diff2' })
|
79
|
+
allow(cli).to receive(:initialize_client).and_return(client_double)
|
80
|
+
end
|
81
|
+
|
82
|
+
subject { cli.restore }
|
83
|
+
|
84
|
+
example 'starts interactive restore' do
|
85
|
+
allow($stdin).to receive(:gets).and_return('q')
|
86
|
+
begin
|
87
|
+
expect { subject }.to(
|
88
|
+
output(/\(r\)estore to Datadog, overwrite local changes and \(d\)ownload, \(s\)kip, or \(q\)uit\?/).to_stdout
|
89
|
+
.and(raise_error(SystemExit))
|
90
|
+
)
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
example 'restore' do
|
95
|
+
allow($stdin).to receive(:gets).and_return('r')
|
96
|
+
expect(dashboards).to receive(:update_with_200).with('diffs1', { 'text' => 'diff' })
|
97
|
+
subject
|
98
|
+
end
|
99
|
+
example 'download' do
|
100
|
+
allow($stdin).to receive(:gets).and_return('d')
|
101
|
+
expect(dashboards).to receive(:write_file).with(%({\n "text": "diff2"\n}), "#{tempdir}/dashboards/diffs1.json")
|
102
|
+
subject
|
103
|
+
end
|
104
|
+
example 'skip' do
|
105
|
+
allow($stdin).to receive(:gets).and_return('s')
|
106
|
+
expect(dashboards).to_not receive(:write_file)
|
107
|
+
expect(dashboards).to_not receive(:update)
|
108
|
+
subject
|
109
|
+
end
|
110
|
+
example 'quit' do
|
111
|
+
allow($stdin).to receive(:gets).and_return('q')
|
112
|
+
expect(dashboards).to_not receive(:write_file)
|
113
|
+
expect(dashboards).to_not receive(:update)
|
114
|
+
expect { subject }.to raise_error(SystemExit)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe DatadogBackup::Core do
|
4
|
+
let(:api_service_double) { double(Dogapi::APIService) }
|
5
|
+
let(:client_double) { double }
|
6
|
+
let(:tempdir) { Dir.mktmpdir }
|
7
|
+
let(:core) do
|
8
|
+
DatadogBackup::Core.new(
|
9
|
+
action: 'backup',
|
10
|
+
api_service: api_service_double,
|
11
|
+
client: client_double,
|
12
|
+
backup_dir: tempdir,
|
13
|
+
diff_format: nil,
|
14
|
+
resources: [],
|
15
|
+
output_format: :json,
|
16
|
+
logger: Logger.new('/dev/null')
|
17
|
+
)
|
18
|
+
end
|
19
|
+
|
20
|
+
describe '#client' do
|
21
|
+
subject { core.client }
|
22
|
+
it { is_expected.to eq client_double }
|
23
|
+
end
|
24
|
+
|
25
|
+
describe '#client_with_200' do
|
26
|
+
subject { core.client_with_200(:get_all_boards) }
|
27
|
+
|
28
|
+
context 'with 200' do
|
29
|
+
before(:example) do
|
30
|
+
allow(client_double).to receive(:get_all_boards).and_return(['200', { foo: :bar }])
|
31
|
+
end
|
32
|
+
|
33
|
+
it { is_expected.to eq({ foo: :bar }) }
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'with not 200' do
|
37
|
+
before(:example) do
|
38
|
+
allow(client_double).to receive(:get_all_boards).and_return(['401', {}])
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'raises an error' do
|
42
|
+
expect { subject }.to raise_error(RuntimeError)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe '#diff' do
|
48
|
+
before(:example) do
|
49
|
+
allow(core).to receive(:get_by_id).and_return({ 'text' => 'diff1', 'extra' => 'diff1' })
|
50
|
+
core.write_file('{"text": "diff2", "extra": "diff2"}', "#{tempdir}/core/diff.json")
|
51
|
+
end
|
52
|
+
|
53
|
+
context 'without banlist' do
|
54
|
+
subject { core.diff('diff') }
|
55
|
+
it {
|
56
|
+
is_expected.to eq <<~EOF
|
57
|
+
---
|
58
|
+
-extra: diff1
|
59
|
+
-text: diff1
|
60
|
+
+extra: diff2
|
61
|
+
+text: diff2
|
62
|
+
EOF
|
63
|
+
}
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'with banlist' do
|
67
|
+
subject { core.diff('diff', ['extra']) }
|
68
|
+
it {
|
69
|
+
is_expected.to eq <<~EOF
|
70
|
+
---
|
71
|
+
-text: diff1
|
72
|
+
+text: diff2
|
73
|
+
EOF
|
74
|
+
}
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
describe '#except' do
|
79
|
+
subject { core.except({ a: :b, b: :c }, [:b]) }
|
80
|
+
it { is_expected.to eq({ a: :b }) }
|
81
|
+
end
|
82
|
+
|
83
|
+
describe '#initialize' do
|
84
|
+
subject { core }
|
85
|
+
it 'makes the subdirectories' do
|
86
|
+
expect(FileUtils).to receive(:mkdir_p).with("#{tempdir}/core")
|
87
|
+
subject
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
describe '#myclass' do
|
92
|
+
subject { core.myclass }
|
93
|
+
it { is_expected.to eq 'core' }
|
94
|
+
end
|
95
|
+
|
96
|
+
describe '#update' do
|
97
|
+
subject { core.update_with_200('abc-123-def', '{"a": "b"}') }
|
98
|
+
example 'it calls Dogapi::APIService.request' do
|
99
|
+
stub_const('Dogapi::APIService::API_VERSION', 'v1')
|
100
|
+
allow(core).to receive(:api_service).and_return(api_service_double)
|
101
|
+
allow(core).to receive(:api_version).and_return('v1')
|
102
|
+
allow(core).to receive(:api_resource_name).and_return('dashboard')
|
103
|
+
expect(api_service_double).to receive(:request).with(Net::HTTP::Put, '/api/v1/dashboard/abc-123-def', nil, '{"a": "b"}', true).and_return(%w[200 Created])
|
104
|
+
subject
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|