datadog_backup 3.0.0.alpha.2 → 3.1.1

data/lib/datadog_backup/resources.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'diffy'
+require 'deepsort'
+require 'faraday'
+require 'faraday/retry'
+
+module DatadogBackup
+  # The default options for backing up and restores.
+  # This base class is meant to be extended by specific resources, such as Dashboards, Monitors, and so on.
+  class Resources
+    include ::DatadogBackup::LocalFilesystem
+    include ::DatadogBackup::Options
+
+    RETRY_OPTIONS = {
+      max: 5,
+      interval: 0.05,
+      interval_randomness: 0.5,
+      backoff_factor: 2
+    }.freeze
+
+    def backup
+      raise 'subclass is expected to implement #backup'
+    end
+
+    # Returns the diffy diff.
+    # Optionally, supply an array of keys to remove from comparison
+    def diff(id)
+      current = except(get_by_id(id)).deep_sort.to_yaml
+      filesystem = except(load_from_file_by_id(id)).deep_sort.to_yaml
+      result = ::Diffy::Diff.new(current, filesystem, include_plus_and_minus_in_html: true).to_s(diff_format)
+      LOGGER.debug("Compared ID #{id} and found filesystem: #{filesystem} <=> current: #{current} == result: #{result}")
+      result.chomp
+    end
+
+    # Returns a hash with banlist elements removed
+    def except(hash)
+      hash.tap do # tap returns self
+        @banlist.each do |key|
+          hash.delete(key) # delete returns the value at the deleted key, hence the tap wrapper
+        end
+      end
+    end
+
+    # Fetch the specified resource from Datadog
+    def get(id)
+      params = {}
+      headers = {}
+      response = api_service.get("/api/#{api_version}/#{api_resource_name}/#{id}", params, headers)
+      body_with_2xx(response)
+    end
+
+    # Returns a list of all resources in Datadog
+    # Do not use directly, but use the child classes' #all method instead
+    def get_all
+      return @get_all if @get_all
+
+      params = {}
+      headers = {}
+      response = api_service.get("/api/#{api_version}/#{api_resource_name}", params, headers)
+      @get_all = body_with_2xx(response)
+    end
+
+    # Download the resource from Datadog and write it to a file
+    def get_and_write_file(id)
+      body = get_by_id(id)
+      write_file(dump(body), filename(id))
+      body
+    end
+
+    # Fetch the specified resource from Datadog and remove the banlist elements
+    def get_by_id(id)
+      except(get(id))
+    end
+
+    def initialize(options)
+      @options = options
+      @banlist = []
+      ::FileUtils.mkdir_p(mydir)
+    end
+
+    def myclass
+      self.class.to_s.split(':').last.downcase
+    end
+
+    # Create a new resource in Datadog
+    def create(body)
+      headers = {}
+      response = api_service.post("/api/#{api_version}/#{api_resource_name}", body, headers)
+      body = body_with_2xx(response)
+      LOGGER.warn "Successfully created #{body.fetch(id_keyname)} in datadog."
+      LOGGER.info 'Invalidating cache'
+      @get_all = nil
+      body
+    end
+
+    # Update an existing resource in Datadog
+    def update(id, body)
+      headers = {}
+      response = api_service.put("/api/#{api_version}/#{api_resource_name}/#{id}", body, headers)
+      body = body_with_2xx(response)
+      LOGGER.warn "Successfully restored #{id} to datadog."
+      LOGGER.info 'Invalidating cache'
+      @get_all = nil
+      body
+    end
+
+    # If the resource exists in Datadog, update it. Otherwise, create it.
+    def restore(id)
+      body = load_from_file_by_id(id)
+      begin
+        update(id, body)
+      rescue RuntimeError => e
+        raise e.message unless e.message.include?('update failed with error 404')
+
+        create_newly(id, body)
+      end
+    end
+
+    # Return the Faraday body from a response with a 2xx status code, otherwise raise an error
+    def body_with_2xx(response)
+      unless response.status.to_s =~ /^2/
+        raise "#{caller_locations(1,
+                                  1)[0].label} failed with error #{response.status}"
+      end
+
+      response.body
+    end
+
+    private
+
+    def api_url
+      ENV.fetch('DD_SITE_URL', 'https://api.datadoghq.com/')
+    end
+
+    def api_version
+      raise 'subclass is expected to implement #api_version'
+    end
+
+    def api_resource_name
+      raise 'subclass is expected to implement #api_resource_name'
+    end
+
+    # Some resources have a different key for the id.
+    def id_keyname
+      'id'
+    end
+
+    def api_service
+      @api_service ||= Faraday.new(
+        url: api_url,
+        headers: {
+          'DD-API-KEY' => ENV.fetch('DD_API_KEY'),
+          'DD-APPLICATION-KEY' => ENV.fetch('DD_APP_KEY')
+        }
+      ) do |faraday|
+        faraday.request :json
+        faraday.request :retry, RETRY_OPTIONS
+        faraday.response(:logger, LOGGER, { headers: true, bodies: LOGGER.debug?, log_level: :debug }) do |logger|
+          logger.filter(/(DD-API-KEY:)([^&]+)/, '\1[REDACTED]')
+          logger.filter(/(DD-APPLICATION-KEY:)([^&]+)/, '\1[REDACTED]')
+        end
+        faraday.response :raise_error
+        faraday.response :json
+        faraday.adapter Faraday.default_adapter
+      end
+    end
+
+    # Create a new resource in Datadog, then move the old file to the new resource's ID
+    def create_newly(file_id, body)
+      new_id = create(body).fetch(id_keyname)
+      FileUtils.rm(find_file_by_id(file_id))
+      get_and_write_file(new_id)
+    end
+  end
+end

data/lib/datadog_backup/synthetics.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module DatadogBackup
+  # Synthetic specific overrides for backup and restore.
+  class Synthetics < Resources
+    def all
+      get_all.fetch('tests')
+    end
+
+    def backup
+      all.map do |synthetic|
+        id = synthetic[id_keyname]
+        get_and_write_file(id)
+      end
+    end
+
+    def get_by_id(id)
+      synthetic = all.select { |s| s[id_keyname].to_s == id.to_s }.first
+      synthetic.nil? ? {} : except(synthetic)
+    end
+
+    def initialize(options)
+      super(options)
+      @banlist = %w[creator created_at modified_at monitor_id public_id].freeze
+    end
+
+    def create(body)
+      create_api_resource_name = api_resource_name(body)
+      headers = {}
+      response = api_service.post("/api/#{api_version}/#{create_api_resource_name}", body, headers)
+      resbody = body_with_2xx(response)
+      LOGGER.warn "Successfully created #{resbody.fetch(id_keyname)} in datadog."
+      LOGGER.info 'Invalidating cache'
+      @get_all = nil
+      resbody
+    end
+
+    def update(id, body)
+      update_api_resource_name = api_resource_name(body)
+      headers = {}
+      response = api_service.put("/api/#{api_version}/#{update_api_resource_name}/#{id}", body, headers)
+      resbody = body_with_2xx(response)
+      LOGGER.warn "Successfully restored #{id} to datadog."
+      LOGGER.info 'Invalidating cache'
+      @get_all = nil
+      resbody
+    end
+
+    private
+
+    def api_version
+      'v1'
+    end
+
+    def api_resource_name(body = nil)
+      return 'synthetics/tests' if body.nil?
+      return 'synthetics/tests' if body['type'].nil?
+      return 'synthetics/tests/browser' if body['type'].to_s == 'browser'
+      return 'synthetics/tests/api' if body['type'].to_s == 'api'
+
+      raise "Unknown type #{body['type']}"
+    end
+
+    def id_keyname
+      'public_id'
+    end
+  end
+end

data/lib/datadog_backup/thread_pool.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module DatadogBackup
+  # Used by CLI and Dashboards to size thread pool according to available CPU resourcess.
   module ThreadPool
     TPOOL = ::Concurrent::ThreadPoolExecutor.new(
       min_threads: [2, Concurrent.processor_count].max,

data/lib/datadog_backup/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DatadogBackup
-  VERSION = '3.0.0.alpha.2'
+  VERSION = '3.1.1'
 end

data/lib/datadog_backup.rb

@@ -5,15 +5,15 @@ require 'concurrent'
 require_relative 'datadog_backup/local_filesystem'
 require_relative 'datadog_backup/options'
 require_relative 'datadog_backup/cli'
-require_relative 'datadog_backup/core'
+require_relative 'datadog_backup/resources'
 require_relative 'datadog_backup/dashboards'
 require_relative 'datadog_backup/monitors'
+require_relative 'datadog_backup/synthetics'
 require_relative 'datadog_backup/thread_pool'
 require_relative 'datadog_backup/version'
 require_relative 'datadog_backup/deprecations'
 DatadogBackup::Deprecations.check
 
-
+# DatadogBackup is a gem for backing up and restoring Datadog monitors and dashboards.
 module DatadogBackup
 end
-

data/release.config.js

@@ -1,4 +1,8 @@
 module.exports = {
+  "branches": [
+    '+([0-9])?(.{+([0-9]),x}).x',
+    'main'
+  ],
   "plugins": [
     "@semantic-release/commit-analyzer",
     "@semantic-release/release-notes-generator",

data/spec/datadog_backup/cli_spec.rb

@@ -29,16 +29,14 @@ describe DatadogBackup::Cli do
   describe '#backup' do
     context 'when dashboards are deleted in datadog' do
       let(:all_dashboards) do
-        [
-          200,
-          {},
+        respond_with200(
           {
             'dashboards' => [
               { 'id' => 'stillthere' },
               { 'id' => 'alsostillthere' }
             ]
           }
-        ]
+        )
       end
 
       before do
@@ -47,8 +45,8 @@ describe DatadogBackup::Cli do
         dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/deleted.json")
 
         stubs.get('/api/v1/dashboard') { all_dashboards }
-        stubs.get('/api/v1/dashboard/stillthere') {[200, {}, {}]}
-        stubs.get('/api/v1/dashboard/alsostillthere') {[200, {}, {}]}
+        stubs.get('/api/v1/dashboard/stillthere') { respond_with200({}) }
+        stubs.get('/api/v1/dashboard/alsostillthere') { respond_with200({}) }
       end
 
       it 'deletes the file locally as well' do
@@ -58,66 +56,89 @@ describe DatadogBackup::Cli do
     end
   end
 
-  describe '#diffs' do
-    subject { cli.diffs }
-
-    before do
-      dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs1.json")
-      dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs2.json")
-      dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs3.json")
-      allow(dashboards).to receive(:get_by_id).and_return({ 'text' => 'diff2' })
-    end
-
-    it {
-      expect(subject).to include(
-        " ---\n id: diffs1\n ---\n-text: diff2\n+text: diff\n",
-        " ---\n id: diffs3\n ---\n-text: diff2\n+text: diff\n",
-        " ---\n id: diffs2\n ---\n-text: diff2\n+text: diff\n"
-      )
-    }
-  end
-
   describe '#restore' do
-    subject { cli.restore }
+    subject(:restore) { cli.restore }
+    let(:stdin) { class_double('STDIN') }
 
+    after(:all) do
+      $stdin = STDIN
+    end
+    
     before do
+      $stdin = stdin
       dashboards.write_file('{"text": "diff"}', "#{tempdir}/dashboards/diffs1.json")
       allow(dashboards).to receive(:get_by_id).and_return({ 'text' => 'diff2' })
+      allow(dashboards).to receive(:write_file)
+      allow(dashboards).to receive(:update)
     end
 
     example 'starts interactive restore' do
-      allow($stdin).to receive(:gets).and_return('q')
+      allow(stdin).to receive(:gets).and_return('q')
 
-      expect { subject }.to(
+      expect { restore }.to(
         output(/\(r\)estore to Datadog, overwrite local changes and \(d\)ownload, \(s\)kip, or \(q\)uit\?/).to_stdout
         .and(raise_error(SystemExit))
       )
     end
 
-    example 'restore' do
-      allow($stdin).to receive(:gets).and_return('r')
-      expect(dashboards).to receive(:update).with('diffs1', { 'text' => 'diff' })
-      subject
+    context 'when the user chooses to restore' do
+      before do
+        allow(stdin).to receive(:gets).and_return('r')
+      end
+
+      example 'it restores from disk to server' do
+        restore
+        expect(dashboards).to have_received(:update).with('diffs1', { 'text' => 'diff' })
+      end
     end
 
-    example 'download' do
-      allow($stdin).to receive(:gets).and_return('d')
-      expect(dashboards).to receive(:write_file).with(%({\n  "text": "diff2"\n}), "#{tempdir}/dashboards/diffs1.json")
-      subject
+    context 'when the user chooses to download' do
+      before do
+        allow(stdin).to receive(:gets).and_return('d')
+      end
+
+      example 'it writes from server to disk' do
+        restore
+        expect(dashboards).to have_received(:write_file).with(%({\n  "text": "diff2"\n}), "#{tempdir}/dashboards/diffs1.json")
+      end
     end
 
-    example 'skip' do
-      allow($stdin).to receive(:gets).and_return('s')
-      expect(dashboards).not_to receive(:write_file)
-      expect(dashboards).not_to receive(:update)
-      subject
+    context 'when the user chooses to skip' do
+      before do
+        allow(stdin).to receive(:gets).and_return('s')
+      end
+
+      example 'it does not write to disk' do
+        restore
+        expect(dashboards).not_to have_received(:write_file)
+      end
+
+      example 'it does not update the server' do
+        restore
+        expect(dashboards).not_to have_received(:update)
+      end
     end
 
-    example 'quit' do
-      allow($stdin).to receive(:gets).and_return('q')
-      expect(dashboards).not_to receive(:write_file)
-      expect(dashboards).not_to receive(:update)
-      expect { subject }.to raise_error(SystemExit)
+    context 'when the user chooses to quit' do
+      before do
+        allow(stdin).to receive(:gets).and_return('q')
+      end
+
+      example 'it exits' do
+        expect { restore }.to raise_error(SystemExit)
+      end
+
+      example 'it does not write to disk' do
+        restore
+      rescue SystemExit
+        expect(dashboards).not_to have_received(:write_file)
+      end
+
+      example 'it does not update the server' do
+        restore
+      rescue SystemExit
+        expect(dashboards).not_to have_received(:update)
+      end
     end
   end
 end

data/spec/datadog_backup/core_spec.rb

@@ -2,133 +2,154 @@
 
 require 'spec_helper'
 
-describe DatadogBackup::Core do
+describe DatadogBackup::Resources do
   let(:stubs) { Faraday::Adapter::Test::Stubs.new }
   let(:api_client_double) { Faraday.new { |f| f.adapter :test, stubs } }
   let(:tempdir) { Dir.mktmpdir }
-  let(:core) do
-    core = described_class.new(
+  let(:resources) do
+    resources = described_class.new(
       action: 'backup',
       backup_dir: tempdir,
       diff_format: nil,
       resources: [],
       output_format: :json
     )
-    allow(core).to receive(:api_service).and_return(api_client_double)
-    return core
+    allow(resources).to receive(:api_service).and_return(api_client_double)
+    return resources
   end
 
-
-
-
   describe '#diff' do
-    subject { core.diff('diff') }
+    subject(:diff) { resources.diff('diff') }
 
     before do
-      allow(core).to receive(:get_by_id).and_return({ 'text' => 'diff1', 'extra' => 'diff1' })
-      core.write_file('{"text": "diff2", "extra": "diff2"}', "#{tempdir}/core/diff.json")
+      allow(resources).to receive(:get_by_id).and_return({ 'text' => 'diff1', 'extra' => 'diff1' })
+      resources.write_file('{"text": "diff2", "extra": "diff2"}', "#{tempdir}/resources/diff.json")
     end
 
     it {
-      expect(subject).to eq <<~EOF
+      expect(diff).to eq(<<~EODIFF
          ---
         -extra: diff1
         -text: diff1
         +extra: diff2
         +text: diff2
-      EOF
+      EODIFF
+      .chomp)
     }
   end
 
   describe '#except' do
-    subject { core.except({ a: :b, b: :c }) }
+    subject { resources.except({ a: :b, b: :c }) }
 
     it { is_expected.to eq({ a: :b, b: :c }) }
   end
 
   describe '#initialize' do
-    subject { core }
+    subject(:myresources) { resources }
 
     it 'makes the subdirectories' do
-      expect(FileUtils).to receive(:mkdir_p).with("#{tempdir}/core")
-      subject
+      fileutils = class_double(FileUtils).as_stubbed_const
+      allow(fileutils).to receive(:mkdir_p)
+      myresources
+      expect(fileutils).to have_received(:mkdir_p).with("#{tempdir}/resources")
     end
   end
 
   describe '#myclass' do
-    subject { core.myclass }
+    subject { resources.myclass }
 
-    it { is_expected.to eq 'core' }
+    it { is_expected.to eq 'resources' }
   end
 
   describe '#create' do
-    subject { core.create({ 'a' => 'b' }) }
+    subject(:create) { resources.create({ 'a' => 'b' }) }
 
     example 'it will post /api/v1/dashboard' do
-      allow(core).to receive(:api_version).and_return('v1')
-      allow(core).to receive(:api_resource_name).and_return('dashboard')
-      stubs.post('/api/v1/dashboard', {'a' => 'b'}) {[200, {},  {'id' => 'whatever-id-abc' }]}
-      subject
+      allow(resources).to receive(:api_version).and_return('v1')
+      allow(resources).to receive(:api_resource_name).and_return('dashboard')
+      stubs.post('/api/v1/dashboard', { 'a' => 'b' }) {  respond_with200({ 'id' => 'whatever-id-abc' }) }
+      create
       stubs.verify_stubbed_calls
     end
   end
 
   describe '#update' do
-    subject { core.update('abc-123-def', { 'a' => 'b' }) }
+    subject(:update) { resources.update('abc-123-def', { 'a' => 'b' }) }
 
     example 'it puts /api/v1/dashboard' do
-      allow(core).to receive(:api_version).and_return('v1')
-      allow(core).to receive(:api_resource_name).and_return('dashboard')
-      stubs.put('/api/v1/dashboard/abc-123-def', {'a' => 'b'}) {[200, {},  {'id' => 'whatever-id-abc' }]}
-      subject
+      allow(resources).to receive(:api_version).and_return('v1')
+      allow(resources).to receive(:api_resource_name).and_return('dashboard')
+      stubs.put('/api/v1/dashboard/abc-123-def', { 'a' => 'b' }) { respond_with200({ 'id' => 'whatever-id-abc' }) }
+      update
       stubs.verify_stubbed_calls
     end
 
     context 'when the id is not found' do
       before do
-        allow(core).to receive(:api_version).and_return('v1')
-        allow(core).to receive(:api_resource_name).and_return('dashboard')
-        stubs.put('/api/v1/dashboard/abc-123-def', {'a' => 'b'}) {[404, {},  {'id' => 'whatever-id-abc' }]}
+        allow(resources).to receive(:api_version).and_return('v1')
+        allow(resources).to receive(:api_resource_name).and_return('dashboard')
+        stubs.put('/api/v1/dashboard/abc-123-def', { 'a' => 'b' }) { [404, {}, { 'id' => 'whatever-id-abc' }] }
       end
+
       it 'raises an error' do
-        expect { subject }.to raise_error(RuntimeError, 'update failed with error 404')
+        expect { update }.to raise_error(RuntimeError, 'update failed with error 404')
       end
     end
   end
 
   describe '#restore' do
     before do
-      allow(core).to receive(:api_version).and_return('api-version-string')
-      allow(core).to receive(:api_resource_name).and_return('api-resource-name-string')
-      stubs.get('/api/api-version-string/api-resource-name-string/abc-123-def') {[200, {},  {'test'  => 'ok' }]}
-      stubs.get('/api/api-version-string/api-resource-name-string/bad-123-id') {[404, {},  {'error'  => 'blahblah_not_found' }]}
-      allow(core).to receive(:load_from_file_by_id).and_return({ 'load' => 'ok' })
+      allow(resources).to receive(:api_version).and_return('api-version-string')
+      allow(resources).to receive(:api_resource_name).and_return('api-resource-name-string')
+      stubs.get('/api/api-version-string/api-resource-name-string/abc-123-def') { respond_with200({ 'test' => 'ok' }) }
+      stubs.get('/api/api-version-string/api-resource-name-string/bad-123-id') do
+        [404, {}, { 'error' => 'blahblah_not_found' }]
+      end
+      allow(resources).to receive(:load_from_file_by_id).and_return({ 'load' => 'ok' })
     end
 
     context 'when id exists' do
-      subject { core.restore('abc-123-def') }
+      subject(:restore) { resources.restore('abc-123-def') }
 
       example 'it calls out to update' do
-        expect(core).to receive(:update).with('abc-123-def', { 'load' => 'ok' })
-        subject
+        allow(resources).to receive(:update)
+        restore
+        expect(resources).to have_received(:update).with('abc-123-def', { 'load' => 'ok' })
       end
     end
 
     context 'when id does not exist on remote' do
-      subject { core.restore('bad-123-id') }
+      subject(:restore_newly) { resources.restore('bad-123-id') }
+
+      let(:fileutils) { class_double(FileUtils).as_stubbed_const }
 
       before do
-        allow(core).to receive(:load_from_file_by_id).and_return({ 'load' => 'ok' })
-        stubs.put('/api/api-version-string/api-resource-name-string/bad-123-id') {[404, {},  {'error'  => 'id not found' }]}
-        stubs.post('/api/api-version-string/api-resource-name-string', {'load' => 'ok'}) {[200, {},  {'id' => 'my-new-id' }]}
+        allow(resources).to receive(:load_from_file_by_id).and_return({ 'load' => 'ok' })
+        stubs.put('/api/api-version-string/api-resource-name-string/bad-123-id') do
+          [404, {}, { 'error' => 'id not found' }]
+        end
+        stubs.post('/api/api-version-string/api-resource-name-string', { 'load' => 'ok' }) do
+          respond_with200({ 'id' => 'my-new-id' })
+        end
+        allow(fileutils).to receive(:rm)
+        allow(resources).to receive(:create).with({ 'load' => 'ok' }).and_return({ 'id' => 'my-new-id' })
+        allow(resources).to receive(:get_and_write_file)
+        allow(resources).to receive(:find_file_by_id).with('bad-123-id').and_return('/path/to/bad-123-id.json')
+      end
+
+      example 'it calls out to create' do
+        restore_newly
+        expect(resources).to have_received(:create).with({ 'load' => 'ok' })
+      end
+
+      example 'it saves the new file' do
+        restore_newly
+        expect(resources).to have_received(:get_and_write_file).with('my-new-id')
       end
 
-      example 'it calls out to create then saves the new file and deletes the new file' do
-        expect(core).to receive(:create).with({ 'load' => 'ok' }).and_return({ 'id' => 'my-new-id' })
-        expect(core).to receive(:get_and_write_file).with('my-new-id')
-        allow(core).to receive(:find_file_by_id).with('bad-123-id').and_return('/path/to/bad-123-id.json')
-        expect(FileUtils).to receive(:rm).with('/path/to/bad-123-id.json')
-        subject
+      example 'it deletes the old file' do
+        restore_newly
+        expect(fileutils).to have_received(:rm).with('/path/to/bad-123-id.json')
       end
     end
   end