datadog_backup 0.9.0

data/lib/datadog_backup/local_filesystem.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'yaml'
+require 'deepsort'
+
+module DatadogBackup
+  module LocalFilesystem
+    ##
+    # Meant to be mixed into DatadogBackup::Core
+    # Relies on @options[:backup_dir] and @options[:output_format]
+
+    def all_files
+      ::Dir.glob(::File.join(backup_dir, '**', '*')).select { |f| ::File.file?(f) }
+    end
+
+    def all_file_ids
+      all_files.map { |file| ::File.basename(file, '.*') }
+    end
+
+    def all_file_ids_for_selected_resources
+      all_file_ids.select do |id|
+        resources.include? class_from_id(id)
+      end
+    end
+
+    def class_from_id(id)
+      class_string = ::File.dirname(find_file_by_id(id)).split('/').last.capitalize
+      ::DatadogBackup.const_get(class_string)
+    end
+
+    def dump(object)
+      if output_format == :json
+        JSON.pretty_generate(object.deep_sort)
+      elsif output_format == :yaml
+        YAML.dump(object.deep_sort)
+      else
+        raise 'invalid output_format specified or not specified'
+      end
+    end
+
+    def filename(id)
+      ::File.join(mydir, "#{id}.#{output_format}")
+    end
+
+    def file_type(filepath)
+      ::File.extname(filepath).strip.downcase[1..-1].to_sym
+    end
+
+    def find_file_by_id(id)
+      ::Dir.glob(::File.join(backup_dir, '**', "#{id}.*")).first
+    end
+
+    def load_from_file(string, output_format)
+      if output_format == :json
+        JSON.parse(string)
+      elsif output_format == :yaml
+        YAML.safe_load(string)
+      else
+        raise 'invalid output_format specified or not specified'
+      end
+    end
+
+    def load_from_file_by_id(id)
+      filepath = find_file_by_id(id)
+      load_from_file(::File.read(filepath), file_type(filepath))
+    end
+
+    def mydir
+      ::File.join(backup_dir, myclass)
+    end
+
+    def purge
+      ::FileUtils.rm(::Dir.glob(File.join(mydir, '*')))
+    end
+
+    def write_file(data, filename)
+      logger.info "Backing up #{filename}"
+      file = ::File.open(filename, 'w')
+      file.write(data)
+    ensure
+      file.close
+    end
+  end
+end

data/lib/datadog_backup/monitors.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module DatadogBackup
+  class Monitors < Core
+    API_SERVICE_NAME = :@monitor_svc
+    BANLIST = %w[overall_state overall_state_modified matching_downtimes modified]
+
+    def all_monitors
+      @all_monitors ||= client_with_200(:get_all_monitors)
+    end
+
+    def api_service
+      # The underlying class from Dogapi that talks to datadog
+      client.instance_variable_get(:@monitor_svc)
+    end
+
+    def api_version
+      'v1'
+    end
+
+    def api_resource_name
+      'monitor'
+    end
+
+    def backup
+      all_monitors.map do |monitor|
+        id = monitor['id']
+        write_file(dump(get_by_id(id)), filename(id))
+      end
+    end
+
+    def diff(id)
+      super(id, BANLIST)
+    end
+
+    def get_by_id(id)
+      except(all_monitors.select { |monitor| monitor['id'].to_s == id.to_s }.first, BANLIST)
+    end
+
+    def restore!; end
+  end
+end

data/lib/datadog_backup/options.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module DatadogBackup
+  module Options
+    def action
+      @options[:action]
+    end
+
+    def backup_dir
+      @options[:backup_dir]
+    end
+
+    def client
+      @options[:client]
+    end
+
+    def concurrency_limit
+      @options[:concurrency_limit] | 2
+    end
+
+    def datadog_api_key
+      @options[:datadog_api_key]
+    end
+
+    def datadog_app_key
+      @options[:datadog_app_key]
+    end
+
+    def diff_format
+      @options[:diff_format]
+    end
+
+    def logger
+      @options[:logger]
+    end
+
+    # Either :json or :yaml
+    def output_format
+      @options[:output_format]
+    end
+
+    def resources
+      @options[:resources]
+    end
+  end
+end

data/lib/datadog_backup/thread_pool.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DatadogBackup
+  module ThreadPool
+    TPOOL = ::Concurrent::ThreadPoolExecutor.new(
+      min_threads: [2, Concurrent.processor_count].max,
+      max_threads: [2, Concurrent.processor_count].max * 2,
+      max_queue: [2, Concurrent.processor_count].max * 512,
+      fallback_policy: :abort
+    )
+
+    def self.watcher(logger)
+      Thread.new(TPOOL) do |pool|
+        while pool.queue_length > 0
+          sleep 2
+          logger.info("#{pool.queue_length} tasks remaining for execution.")
+        end
+      end
+    end
+
+    def self.shutdown(logger)
+      logger.fatal 'Shutdown signal caught. Performing orderly shut down of thread pool. Press Ctrl+C again to forcibly shut down, but be warned, DATA LOSS MAY OCCUR.'
+      TPOOL.shutdown
+      TPOOL.wait_for_termination
+    rescue SystemExit, Interrupt
+      logger.fatal 'OK Nuking, DATA LOSS MAY OCCUR.'
+      TPOOL.kill
+    end
+  end
+end

data/lib/datadog_backup/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DatadogBackup
+  VERSION = '0.9.0'
+end

data/release.config.js

@@ -0,0 +1,43 @@
+module.exports = {
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+    [
+      "@semantic-release/changelog",
+      {
+        "changelogFile": "CHANGELOG.md"
+      }
+    ],
+    [
+      "semantic-release-rubygem",
+      {
+        "gemFileDir": "."
+      }
+    ],
+    [
+      "@semantic-release/git",
+      {
+        "assets": [
+          "CHANGELOG.md"
+        ],
+        "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+      }
+    ],
+    [
+      "@semantic-release/github",
+      {
+        "assets": [
+          {
+            "path": "datadog_backup.zip",
+            "name": "datadog_backup.${nextRelease.version}.zip",
+            "label": "Full zip distribution"
+          },
+          {
+            "path": "datadog_backup-*.gem",
+            "label": "Gem distribution"
+          }
+        ]
+      }
+    ],
+  ]
+};

data/spec/datadog_backup/cli_spec.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe DatadogBackup::Cli do
+  let(:client_double) { double }
+  let(:tempdir) { Dir.mktmpdir }
+  let(:options) do
+    {
+      action: 'backup',
+      backup_dir: tempdir,
+      client: client_double,
+      datadog_api_key: 1,
+      datadog_app_key: 1,
+      diff_format: nil,
+      logger: Logger.new('/dev/null'),
+      output_format: :json,
+      resources: [DatadogBackup::Dashboards]
+    }
+  end
+  let(:cli) { DatadogBackup::Cli.new(options) }
+  let(:dashboards) { DatadogBackup::Dashboards.new(options) }
+
+  before(:example) do
+    allow(cli).to receive(:resource_instances).and_return([dashboards])
+  end
+
+  describe '#backup' do
+    context 'when dashboards are deleted in datadog' do
+      let(:all_boards) do
+        [
+          '200',
+          {
+            'dashboards' => [
+              { 'id' => 'stillthere' },
+              { 'id' => 'alsostillthere' }
+            ]
+          }
+        ]
+      end
+      before(:example) do
+        dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/stillthere.json")
+        dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/alsostillthere.json")
+        dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/deleted.json")
+
+        allow(client_double).to receive(:get_all_boards).and_return(all_boards)
+        allow(client_double).to receive(:get_board).and_return(['200', {}])
+      end
+
+      it 'deletes the file locally as well' do
+        cli.backup
+        expect { File.open("#{tempdir}/dashboards/deleted.json", 'r') }.to raise_error(Errno::ENOENT)
+      end
+    end
+  end
+
+  describe '#diffs' do
+    before(:example) do
+      dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs1.json")
+      dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs2.json")
+      dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs3.json")
+      allow(dashboards).to receive(:get_by_id).and_return({ 'text' => 'diff2' })
+      allow(cli).to receive(:initialize_client).and_return(client_double)
+    end
+    subject { cli.diffs }
+    it {
+      is_expected.to include(
+        " ---\n id: diffs1\n ---\n-text: diff2\n+text: diff\n",
+        " ---\n id: diffs3\n ---\n-text: diff2\n+text: diff\n",
+        " ---\n id: diffs2\n ---\n-text: diff2\n+text: diff\n"
+      )
+    }
+  end
+
+  describe '#restore' do
+    before(:example) do
+      dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs1.json")
+      allow(dashboards).to receive(:get_by_id).and_return({ 'text' => 'diff2' })
+      allow(cli).to receive(:initialize_client).and_return(client_double)
+    end
+
+    subject { cli.restore }
+
+    example 'starts interactive restore' do
+      allow($stdin).to receive(:gets).and_return('q')
+      begin
+        expect { subject }.to(
+          output(/\(r\)estore to Datadog, overwrite local changes and \(d\)ownload, \(s\)kip, or \(q\)uit\?/).to_stdout
+          .and(raise_error(SystemExit))
+        )
+      end
+    end
+
+    example 'restore' do
+      allow($stdin).to receive(:gets).and_return('r')
+      expect(dashboards).to receive(:update_with_200).with('diffs1', { 'text' => 'diff' })
+      subject
+    end
+    example 'download' do
+      allow($stdin).to receive(:gets).and_return('d')
+      expect(dashboards).to receive(:write_file).with(%({\n  "text": "diff2"\n}), "#{tempdir}/dashboards/diffs1.json")
+      subject
+    end
+    example 'skip' do
+      allow($stdin).to receive(:gets).and_return('s')
+      expect(dashboards).to_not receive(:write_file)
+      expect(dashboards).to_not receive(:update)
+      subject
+    end
+    example 'quit' do
+      allow($stdin).to receive(:gets).and_return('q')
+      expect(dashboards).to_not receive(:write_file)
+      expect(dashboards).to_not receive(:update)
+      expect { subject }.to raise_error(SystemExit)
+    end
+  end
+end

data/spec/datadog_backup/core_spec.rb

@@ -0,0 +1,107 @@
+require 'spec_helper'
+
+describe DatadogBackup::Core do
+  let(:api_service_double) { double(Dogapi::APIService) }
+  let(:client_double) { double }
+  let(:tempdir) { Dir.mktmpdir }
+  let(:core) do
+    DatadogBackup::Core.new(
+      action: 'backup',
+      api_service: api_service_double,
+      client: client_double,
+      backup_dir: tempdir,
+      diff_format: nil,
+      resources: [],
+      output_format: :json,
+      logger: Logger.new('/dev/null')
+    )
+  end
+
+  describe '#client' do
+    subject { core.client }
+    it { is_expected.to eq client_double }
+  end
+
+  describe '#client_with_200' do
+    subject { core.client_with_200(:get_all_boards) }
+
+    context 'with 200' do
+      before(:example) do
+        allow(client_double).to receive(:get_all_boards).and_return(['200', { foo: :bar }])
+      end
+
+      it { is_expected.to eq({ foo: :bar }) }
+    end
+
+    context 'with not 200' do
+      before(:example) do
+        allow(client_double).to receive(:get_all_boards).and_return(['401', {}])
+      end
+
+      it 'raises an error' do
+        expect { subject }.to raise_error(RuntimeError)
+      end
+    end
+  end
+
+  describe '#diff' do
+    before(:example) do
+      allow(core).to receive(:get_by_id).and_return({ 'text' => 'diff1', 'extra' => 'diff1' })
+      core.write_file('{"text": "diff2", "extra": "diff2"}', "#{tempdir}/core/diff.json")
+    end
+
+    context 'without banlist' do
+      subject { core.diff('diff') }
+      it {
+        is_expected.to eq <<~EOF
+           ---
+          -extra: diff1
+          -text: diff1
+          +extra: diff2
+          +text: diff2
+        EOF
+      }
+    end
+
+    context 'with banlist' do
+      subject { core.diff('diff', ['extra']) }
+      it {
+        is_expected.to eq <<~EOF
+           ---
+          -text: diff1
+          +text: diff2
+        EOF
+      }
+    end
+  end
+
+  describe '#except' do
+    subject { core.except({ a: :b, b: :c }, [:b]) }
+    it { is_expected.to eq({ a: :b }) }
+  end
+
+  describe '#initialize' do
+    subject { core }
+    it 'makes the subdirectories' do
+      expect(FileUtils).to receive(:mkdir_p).with("#{tempdir}/core")
+      subject
+    end
+  end
+
+  describe '#myclass' do
+    subject { core.myclass }
+    it { is_expected.to eq 'core' }
+  end
+
+  describe '#update' do
+    subject { core.update_with_200('abc-123-def', '{"a": "b"}') }
+    example 'it calls Dogapi::APIService.request' do
+      stub_const('Dogapi::APIService::API_VERSION', 'v1')
+      allow(core).to receive(:api_service).and_return(api_service_double)
+      allow(core).to receive(:api_version).and_return('v1')
+      allow(core).to receive(:api_resource_name).and_return('dashboard')
+      expect(api_service_double).to receive(:request).with(Net::HTTP::Put, '/api/v1/dashboard/abc-123-def', nil, '{"a": "b"}', true).and_return(%w[200 Created])
+      subject
+    end
+  end
+end