datadog_backup 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,86 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fileutils'
4
+ require 'json'
5
+ require 'yaml'
6
+ require 'deepsort'
7
+
8
+ module DatadogBackup
9
+ module LocalFilesystem
10
+ ##
11
+ # Meant to be mixed into DatadogBackup::Core
12
+ # Relies on @options[:backup_dir] and @options[:output_format]
13
+
14
+ def all_files
15
+ ::Dir.glob(::File.join(backup_dir, '**', '*')).select { |f| ::File.file?(f) }
16
+ end
17
+
18
+ def all_file_ids
19
+ all_files.map { |file| ::File.basename(file, '.*') }
20
+ end
21
+
22
+ def all_file_ids_for_selected_resources
23
+ all_file_ids.select do |id|
24
+ resources.include? class_from_id(id)
25
+ end
26
+ end
27
+
28
+ def class_from_id(id)
29
+ class_string = ::File.dirname(find_file_by_id(id)).split('/').last.capitalize
30
+ ::DatadogBackup.const_get(class_string)
31
+ end
32
+
33
+ def dump(object)
34
+ if output_format == :json
35
+ JSON.pretty_generate(object.deep_sort)
36
+ elsif output_format == :yaml
37
+ YAML.dump(object.deep_sort)
38
+ else
39
+ raise 'invalid output_format specified or not specified'
40
+ end
41
+ end
42
+
43
+ def filename(id)
44
+ ::File.join(mydir, "#{id}.#{output_format}")
45
+ end
46
+
47
+ def file_type(filepath)
48
+ ::File.extname(filepath).strip.downcase[1..-1].to_sym
49
+ end
50
+
51
+ def find_file_by_id(id)
52
+ ::Dir.glob(::File.join(backup_dir, '**', "#{id}.*")).first
53
+ end
54
+
55
+ def load_from_file(string, output_format)
56
+ if output_format == :json
57
+ JSON.parse(string)
58
+ elsif output_format == :yaml
59
+ YAML.safe_load(string)
60
+ else
61
+ raise 'invalid output_format specified or not specified'
62
+ end
63
+ end
64
+
65
+ def load_from_file_by_id(id)
66
+ filepath = find_file_by_id(id)
67
+ load_from_file(::File.read(filepath), file_type(filepath))
68
+ end
69
+
70
+ def mydir
71
+ ::File.join(backup_dir, myclass)
72
+ end
73
+
74
+ def purge
75
+ ::FileUtils.rm(::Dir.glob(File.join(mydir, '*')))
76
+ end
77
+
78
+ def write_file(data, filename)
79
+ logger.info "Backing up #{filename}"
80
+ file = ::File.open(filename, 'w')
81
+ file.write(data)
82
+ ensure
83
+ file.close
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DatadogBackup
4
+ class Monitors < Core
5
+ API_SERVICE_NAME = :@monitor_svc
6
+ BANLIST = %w[overall_state overall_state_modified matching_downtimes modified]
7
+
8
+ def all_monitors
9
+ @all_monitors ||= client_with_200(:get_all_monitors)
10
+ end
11
+
12
+ def api_service
13
+ # The underlying class from Dogapi that talks to datadog
14
+ client.instance_variable_get(:@monitor_svc)
15
+ end
16
+
17
+ def api_version
18
+ 'v1'
19
+ end
20
+
21
+ def api_resource_name
22
+ 'monitor'
23
+ end
24
+
25
+ def backup
26
+ all_monitors.map do |monitor|
27
+ id = monitor['id']
28
+ write_file(dump(get_by_id(id)), filename(id))
29
+ end
30
+ end
31
+
32
+ def diff(id)
33
+ super(id, BANLIST)
34
+ end
35
+
36
+ def get_by_id(id)
37
+ except(all_monitors.select { |monitor| monitor['id'].to_s == id.to_s }.first, BANLIST)
38
+ end
39
+
40
+ def restore!; end
41
+ end
42
+ end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DatadogBackup
4
+ module Options
5
+ def action
6
+ @options[:action]
7
+ end
8
+
9
+ def backup_dir
10
+ @options[:backup_dir]
11
+ end
12
+
13
+ def client
14
+ @options[:client]
15
+ end
16
+
17
+ def concurrency_limit
18
+ @options[:concurrency_limit] | 2
19
+ end
20
+
21
+ def datadog_api_key
22
+ @options[:datadog_api_key]
23
+ end
24
+
25
+ def datadog_app_key
26
+ @options[:datadog_app_key]
27
+ end
28
+
29
+ def diff_format
30
+ @options[:diff_format]
31
+ end
32
+
33
+ def logger
34
+ @options[:logger]
35
+ end
36
+
37
+ # Either :json or :yaml
38
+ def output_format
39
+ @options[:output_format]
40
+ end
41
+
42
+ def resources
43
+ @options[:resources]
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DatadogBackup
4
+ module ThreadPool
5
+ TPOOL = ::Concurrent::ThreadPoolExecutor.new(
6
+ min_threads: [2, Concurrent.processor_count].max,
7
+ max_threads: [2, Concurrent.processor_count].max * 2,
8
+ max_queue: [2, Concurrent.processor_count].max * 512,
9
+ fallback_policy: :abort
10
+ )
11
+
12
+ def self.watcher(logger)
13
+ Thread.new(TPOOL) do |pool|
14
+ while pool.queue_length > 0
15
+ sleep 2
16
+ logger.info("#{pool.queue_length} tasks remaining for execution.")
17
+ end
18
+ end
19
+ end
20
+
21
+ def self.shutdown(logger)
22
+ logger.fatal 'Shutdown signal caught. Performing orderly shut down of thread pool. Press Ctrl+C again to forcibly shut down, but be warned, DATA LOSS MAY OCCUR.'
23
+ TPOOL.shutdown
24
+ TPOOL.wait_for_termination
25
+ rescue SystemExit, Interrupt
26
+ logger.fatal 'OK Nuking, DATA LOSS MAY OCCUR.'
27
+ TPOOL.kill
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DatadogBackup
4
+ VERSION = '0.9.0'
5
+ end
@@ -0,0 +1,43 @@
1
+ module.exports = {
2
+ "plugins": [
3
+ "@semantic-release/commit-analyzer",
4
+ "@semantic-release/release-notes-generator",
5
+ [
6
+ "@semantic-release/changelog",
7
+ {
8
+ "changelogFile": "CHANGELOG.md"
9
+ }
10
+ ],
11
+ [
12
+ "semantic-release-rubygem",
13
+ {
14
+ "gemFileDir": "."
15
+ }
16
+ ],
17
+ [
18
+ "@semantic-release/git",
19
+ {
20
+ "assets": [
21
+ "CHANGELOG.md"
22
+ ],
23
+ "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
24
+ }
25
+ ],
26
+ [
27
+ "@semantic-release/github",
28
+ {
29
+ "assets": [
30
+ {
31
+ "path": "datadog_backup.zip",
32
+ "name": "datadog_backup.${nextRelease.version}.zip",
33
+ "label": "Full zip distribution"
34
+ },
35
+ {
36
+ "path": "datadog_backup-*.gem",
37
+ "label": "Gem distribution"
38
+ }
39
+ ]
40
+ }
41
+ ],
42
+ ]
43
+ };
@@ -0,0 +1,117 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ describe DatadogBackup::Cli do
6
+ let(:client_double) { double }
7
+ let(:tempdir) { Dir.mktmpdir }
8
+ let(:options) do
9
+ {
10
+ action: 'backup',
11
+ backup_dir: tempdir,
12
+ client: client_double,
13
+ datadog_api_key: 1,
14
+ datadog_app_key: 1,
15
+ diff_format: nil,
16
+ logger: Logger.new('/dev/null'),
17
+ output_format: :json,
18
+ resources: [DatadogBackup::Dashboards]
19
+ }
20
+ end
21
+ let(:cli) { DatadogBackup::Cli.new(options) }
22
+ let(:dashboards) { DatadogBackup::Dashboards.new(options) }
23
+
24
+ before(:example) do
25
+ allow(cli).to receive(:resource_instances).and_return([dashboards])
26
+ end
27
+
28
+ describe '#backup' do
29
+ context 'when dashboards are deleted in datadog' do
30
+ let(:all_boards) do
31
+ [
32
+ '200',
33
+ {
34
+ 'dashboards' => [
35
+ { 'id' => 'stillthere' },
36
+ { 'id' => 'alsostillthere' }
37
+ ]
38
+ }
39
+ ]
40
+ end
41
+ before(:example) do
42
+ dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/stillthere.json")
43
+ dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/alsostillthere.json")
44
+ dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/deleted.json")
45
+
46
+ allow(client_double).to receive(:get_all_boards).and_return(all_boards)
47
+ allow(client_double).to receive(:get_board).and_return(['200', {}])
48
+ end
49
+
50
+ it 'deletes the file locally as well' do
51
+ cli.backup
52
+ expect { File.open("#{tempdir}/dashboards/deleted.json", 'r') }.to raise_error(Errno::ENOENT)
53
+ end
54
+ end
55
+ end
56
+
57
+ describe '#diffs' do
58
+ before(:example) do
59
+ dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs1.json")
60
+ dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs2.json")
61
+ dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs3.json")
62
+ allow(dashboards).to receive(:get_by_id).and_return({ 'text' => 'diff2' })
63
+ allow(cli).to receive(:initialize_client).and_return(client_double)
64
+ end
65
+ subject { cli.diffs }
66
+ it {
67
+ is_expected.to include(
68
+ " ---\n id: diffs1\n ---\n-text: diff2\n+text: diff\n",
69
+ " ---\n id: diffs3\n ---\n-text: diff2\n+text: diff\n",
70
+ " ---\n id: diffs2\n ---\n-text: diff2\n+text: diff\n"
71
+ )
72
+ }
73
+ end
74
+
75
+ describe '#restore' do
76
+ before(:example) do
77
+ dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs1.json")
78
+ allow(dashboards).to receive(:get_by_id).and_return({ 'text' => 'diff2' })
79
+ allow(cli).to receive(:initialize_client).and_return(client_double)
80
+ end
81
+
82
+ subject { cli.restore }
83
+
84
+ example 'starts interactive restore' do
85
+ allow($stdin).to receive(:gets).and_return('q')
86
+ begin
87
+ expect { subject }.to(
88
+ output(/\(r\)estore to Datadog, overwrite local changes and \(d\)ownload, \(s\)kip, or \(q\)uit\?/).to_stdout
89
+ .and(raise_error(SystemExit))
90
+ )
91
+ end
92
+ end
93
+
94
+ example 'restore' do
95
+ allow($stdin).to receive(:gets).and_return('r')
96
+ expect(dashboards).to receive(:update_with_200).with('diffs1', { 'text' => 'diff' })
97
+ subject
98
+ end
99
+ example 'download' do
100
+ allow($stdin).to receive(:gets).and_return('d')
101
+ expect(dashboards).to receive(:write_file).with(%({\n "text": "diff2"\n}), "#{tempdir}/dashboards/diffs1.json")
102
+ subject
103
+ end
104
+ example 'skip' do
105
+ allow($stdin).to receive(:gets).and_return('s')
106
+ expect(dashboards).to_not receive(:write_file)
107
+ expect(dashboards).to_not receive(:update)
108
+ subject
109
+ end
110
+ example 'quit' do
111
+ allow($stdin).to receive(:gets).and_return('q')
112
+ expect(dashboards).to_not receive(:write_file)
113
+ expect(dashboards).to_not receive(:update)
114
+ expect { subject }.to raise_error(SystemExit)
115
+ end
116
+ end
117
+ end
@@ -0,0 +1,107 @@
1
+ require 'spec_helper'
2
+
3
+ describe DatadogBackup::Core do
4
+ let(:api_service_double) { double(Dogapi::APIService) }
5
+ let(:client_double) { double }
6
+ let(:tempdir) { Dir.mktmpdir }
7
+ let(:core) do
8
+ DatadogBackup::Core.new(
9
+ action: 'backup',
10
+ api_service: api_service_double,
11
+ client: client_double,
12
+ backup_dir: tempdir,
13
+ diff_format: nil,
14
+ resources: [],
15
+ output_format: :json,
16
+ logger: Logger.new('/dev/null')
17
+ )
18
+ end
19
+
20
+ describe '#client' do
21
+ subject { core.client }
22
+ it { is_expected.to eq client_double }
23
+ end
24
+
25
+ describe '#client_with_200' do
26
+ subject { core.client_with_200(:get_all_boards) }
27
+
28
+ context 'with 200' do
29
+ before(:example) do
30
+ allow(client_double).to receive(:get_all_boards).and_return(['200', { foo: :bar }])
31
+ end
32
+
33
+ it { is_expected.to eq({ foo: :bar }) }
34
+ end
35
+
36
+ context 'with not 200' do
37
+ before(:example) do
38
+ allow(client_double).to receive(:get_all_boards).and_return(['401', {}])
39
+ end
40
+
41
+ it 'raises an error' do
42
+ expect { subject }.to raise_error(RuntimeError)
43
+ end
44
+ end
45
+ end
46
+
47
+ describe '#diff' do
48
+ before(:example) do
49
+ allow(core).to receive(:get_by_id).and_return({ 'text' => 'diff1', 'extra' => 'diff1' })
50
+ core.write_file('{"text": "diff2", "extra": "diff2"}', "#{tempdir}/core/diff.json")
51
+ end
52
+
53
+ context 'without banlist' do
54
+ subject { core.diff('diff') }
55
+ it {
56
+ is_expected.to eq <<~EOF
57
+ ---
58
+ -extra: diff1
59
+ -text: diff1
60
+ +extra: diff2
61
+ +text: diff2
62
+ EOF
63
+ }
64
+ end
65
+
66
+ context 'with banlist' do
67
+ subject { core.diff('diff', ['extra']) }
68
+ it {
69
+ is_expected.to eq <<~EOF
70
+ ---
71
+ -text: diff1
72
+ +text: diff2
73
+ EOF
74
+ }
75
+ end
76
+ end
77
+
78
+ describe '#except' do
79
+ subject { core.except({ a: :b, b: :c }, [:b]) }
80
+ it { is_expected.to eq({ a: :b }) }
81
+ end
82
+
83
+ describe '#initialize' do
84
+ subject { core }
85
+ it 'makes the subdirectories' do
86
+ expect(FileUtils).to receive(:mkdir_p).with("#{tempdir}/core")
87
+ subject
88
+ end
89
+ end
90
+
91
+ describe '#myclass' do
92
+ subject { core.myclass }
93
+ it { is_expected.to eq 'core' }
94
+ end
95
+
96
+ describe '#update' do
97
+ subject { core.update_with_200('abc-123-def', '{"a": "b"}') }
98
+ example 'it calls Dogapi::APIService.request' do
99
+ stub_const('Dogapi::APIService::API_VERSION', 'v1')
100
+ allow(core).to receive(:api_service).and_return(api_service_double)
101
+ allow(core).to receive(:api_version).and_return('v1')
102
+ allow(core).to receive(:api_resource_name).and_return('dashboard')
103
+ expect(api_service_double).to receive(:request).with(Net::HTTP::Put, '/api/v1/dashboard/abc-123-def', nil, '{"a": "b"}', true).and_return(%w[200 Created])
104
+ subject
105
+ end
106
+ end
107
+ end