elasticsearch_s3_backup 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a2652e8865ae1bae268503eacb635d2ce586592
-  data.tar.gz: 11c55edfc00bdc8abb85aff511c939fe0eb4d9e6
+  metadata.gz: f4080ff7b9f8bac6212fe0d240806c810259e6e1
+  data.tar.gz: 9669329c6836926d87e08494e1fc9b7c5f596186
 SHA512:
-  metadata.gz: e7e337aa678f99dba4ff872761039f15fa511c6d7246d955f562d52a46889e83bd0789ae15ef841419c6f2f8f424c0a757b1c829c0992683050fdf077a85f8fb
-  data.tar.gz: c27135eab4c5871cee1d48b0ed6f12f516040bf32049d6ac29dd017db3816223f14d1c91d4286052d03b68abfca4fc04ebe4fb259ad27737441396c024975234
+  metadata.gz: 183937a0de6ea3ad69f945cd484e9e50f3613ece45bac331685d145aa3faf028547e17f8064f5ada11987147f7c27ff70a6b56544ac4d9411af0263183221988
+  data.tar.gz: f81624dffe286fc624630e2c31c439e8f8d01de7d170c3b4e0652803abe3f21410a0f72a9454f92049537469fc4fa9f70e6621869052f828d29f69d6de4c5b1e

data/Rakefile

@@ -1,6 +1,6 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/elasticsearch_s3_backup.gemspec

@@ -23,7 +23,10 @@ Gem::Specification.new do |spec|
          'pushes.'
   end
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
@@ -37,4 +40,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'faker'
   spec.add_dependency 'pagerduty'
   spec.add_dependency 'sentry-raven'
+  spec.add_dependency 'elasticsearch'
 end

data/lib/elasticsearch_s3_backup.rb

@@ -2,160 +2,111 @@ require 'elasticsearch_s3_backup/version'
 require 'active_support/time'
 require 'unirest'
 require 'logger'
-require 'faker'
 require 'pagerduty'
 require 'yaml'
 require 'sentry-raven'
+require 'ostruct'
+require 'elasticsearch'
 
 module EverTools
   class ElasticsearchS3Backup
-    def conf
-      @conf ||= YAML.load_file('/etc/s3_backup.yml')
-    end
-
-    def pagerduty
-      @pagerduty ||= Pagerduty.new conf['pagerduty_api_key']
-    end
+    extend Forwardable
 
-    def configure_sentry!
-      Raven.configure { config.dsn = conf['sentry_dsn'] }
-    end
+    def_delegators :@conf,
+                   :pagerduty_api_key,
+                   :test_size,
+                   :new_repo_params,
+                   :sentry_dsn,
+                   :node_name,
+                   :elasticsearch_auth_file,
+                   :cluster_name
 
-    def auth
-      @auth ||= File.read(conf['elasticsearch_auth_file']).strip.split ':'
-    end
+    attr_reader :conf, :backup_repo
 
     def initialize
-      Unirest.default_header 'Accept', 'application/json'
-      Unirest.default_header 'Content-Type', 'application/json'
-      Unirest.timeout 30
-
-      @url = 'http://localhost:9200'
+      @conf = OpenStruct.new(YAML.load_file('/etc/s3_backup.yml'))
+      Raven.configure { config.dsn = sentry_dsn } if sentry_dsn
 
-      @backup_index  = 'backup_test'
-      @restore_index = 'restore_test'
-
-      now       = Time.new.utc
-      @monthly  = now.strftime '%m-%Y'
-      @datetime = now.strftime '%m-%d_%H%M'
-
-      @monthly_snap_url = "#{@url}/_snapshot/#{@monthly}"
-
-      @backup_timeout = 1.hours
+      now                 = Time.new.utc
+      @backup_test_index  = "backup_test_#{now.to_i}"
+      @restore_test_index = "restore_test_#{now.to_i}"
+      @backup_repo        = now.strftime '%m-%Y'
+      @snapshot_label     = now.strftime '%m-%d_%H%M'
     end
 
     def logger
-      # Set up logging
-      @logger ||= Logger.new(conf['log']).tap do |l|
-        l.level = Logger::INFO
-        l.progname = 's3_backup'
-        l.formatter =
-          proc do |severity, datetime, progname, msg|
-            "#{datetime.utc} [#{progname}] #{severity}: #{msg}\n"
-          end
-      end
+      @logger ||= Logger.new(conf['log']).tap { |l| l.progname = 's3_backup' }
     end
 
-    # rubocop:disable Metrics/AbcSize
-    def es_api(method, uri, params = {})
-      tries = 3
-      begin
-        r = Unirest.send(method, uri, params)
-        case r.code
-        when 200..299
-          return r
-        when 400..499
-          logger.debug "#{method} request to #{uri} received #{r.code} (params: #{params.inspect})\n" \
-                      "Body:\n" \
-                      "#{r.body}\n"
-          return r
-        end
-        # byebug
-        fail "#{method.upcase} request to #{uri} failed (params: #{params.inspect})\n" \
-             "Response code: #{r.code}\n" \
-             "Body:\n" \
-             "#{r.body}\n"
-      rescue RuntimeError => e
-        tries -= 1
-        retry if e.message == 'Request Timeout' && tries > 0
-        raise e
+    def pagerduty
+      @pagerduty ||= Pagerduty.new pagerduty_api_key
+    end
+
+    def notify(e)
+      if conf['env'] == 'prod'
+        pagerduty.trigger(
+          'prod Elasticsearch S3 failed',
+          client: node_name,
+          details: "#{e.message}\n\n#{e.backtrace}"
+        )
       end
+
+      Raven.capture_exception(e) if sentry_dsn
     end
-    # rubocop:enable Metrics/AbcSize
 
-    def master?
-      es_api(:get, "#{@url}/_cat/master").body[0]['node'] == conf['node_name']
+    def auth
+      File.read(elasticsearch_auth_file).strip
     end
 
-    # Check if an index exists
-    def index?(uri)
-      es_api(:get, "#{@url}/#{uri}").code == 200
+    def es_api
+      @es_api ||= begin
+        es_host = @conf['es_host'] || 'localhost'
+        Elasticsearch::Client.new URI "http://#{auth}@#{es_host}:9200"
+      end
     end
 
-    # Check if a backup repo exists
-    def repo?
-      es_api(:get, @monthly_snap_url).code == 200
+    def master?
+      es_api.nodes.info['nodes'][es_api.cluster.state['master_node']]['node'] == node_name
     end
 
-    def notify(e)
-      pagerduty.trigger(
-        'prod Elasticsearch S3 failed',
-        client: conf['node_name'],
-        details: "#{e.message}\n\n#{e.backtrace}"
-      )
+    def pseudo_random_string
+      'a' + rand(10**100).to_s
     end
 
     def insert_test_data
-      # Generate some test data using Faker
-      #
-      # Uses Bitcoin addresses for their random, hash-like nature
-      # Creates the `backup_test` index if necessary
-      # Updates the set of `dummy` documents in the `backup_test` on every run
-
-      logger.info 'Generating test data using Faker…'
-      conf['test_size'].times do |i|
-        es_api(
-          :put,
-          "#{@url}/#{@backup_index}/dummy/#{i}",
-          parameters: {
-            test_value: Faker::Bitcoin.address
-          }.to_json
+      logger.info 'Generating test data using math…'
+      test_size.times do |i|
+        es_api.create(
+          index: @backup_test_index,
+          type: 'dummy',
+          id: i,
+          body: { test_value: pseudo_random_string }
         )
       end
     end
 
-    def delete_index(index)
-      logger.info "Deleting index: #{index}"
-      es_api(
-        :delete,
-        "#{@url}/#{index}",
-        auth: { user: auth.first, password: auth.last }
-      )
-    end
-
     def create_repo
-      new_repo_params = conf['new_repo_params'].merge(
+      new_repo_params = new_repo_params.merge(
         type: 's3',
         settings: {
-          base_path: "/elasticsearch/#{cluster_name}/#{conf['env']}/#{@monthly}",
+          base_path: "/elasticsearch/#{cluster_name}/#{conf['env']}/#{@backup_repo}",
           server_side_encryption: true
         }
       )
 
       logger.info 'Creating a new monthly ES backup repo…'
-      es_api(
-        :put,
-        @monthly_snap_url,
-        parameters: new_repo_params.to_json
-      )
+      es_api.snapshot.create_repository repository: @backup_repo,
+                                        body: new_repo_params
     end
 
     def valid_date?(date)
+      # rubocop:disable Style/RescueModifier
       Time.strptime(date, '%m-%Y') rescue false
+      # rubocop:enable Style/RescueModifier
     end
 
     def dated_repos
-      es_api(:get, "#{@url}/_snapshot").body.keys.select { |r| valid_date? r }
+      es_api.snapshot.get_repository.keys.select { |r| valid_date? r }
     end
 
     def remove_expired_backups
@@ -163,106 +114,42 @@ module EverTools
       logger.info "Removing backups older than #{3.months.ago.strftime '%m-%Y'}"
       dated_repos.select { |b| Time.strptime(b, '%m-%Y') < 3.months.ago }.each do |repo|
         logger.info "Removing #{repo}"
-        es_api :delete, "#{@url}/_snapshot/#{repo}"
+        es_api.snapshot.delete_repository repository: repo
       end
     end
 
-    # rubocop:disable Metrics/AbcSize
-    def snapshot
-      backup_uri = "#{@monthly_snap_url}/#{@datetime}"
-      status_req = es_api :get, "#{backup_uri}/_status"
-      if status_req.code == 404 ||
-         status_req.body['snapshots'].empty?
-        fail "Could not find the backup I just created (#{backup_uri})"
-      end
-      snapshot = status_req.body['snapshots'].first
-      logger.info "Backup state: #{snapshot['state']} " \
-                  "(finished shard #{snapshot['shards_stats']['done']} of " \
-                  "#{snapshot['shards_stats']['total']})"
-    end
-    # rubocop:enable Metrics/AbcSize
-
-    def backup_complete?
-      case snapshot['state']
-      when 'IN_PROGRESS', 'STARTED'
-        return false
-      when 'SUCCESS'
-        return true
-      end
-      fail "Backup failed!\n" \
-           "State: #{snapshot['state']}\n" \
-           "Response Body: #{status_req.body}"
-    end
-
-    def verify_create!
-      # Check the status of the backup for up to an hour
-      backup_start_time = Time.now.utc
-      until Time.now.utc > (backup_start_time + @backup_timeout)
-        return true if backup_complete?
-        # Don't hammer the status endpoint
-        sleep 15
-      end
-
-      fail 'Create timed out'
-    end
-
-    def make_new_backup
+    def create_snapshot
       # Make a backup (full on new month, incremental otherwise)
-      logger.info "Starting a new backup (#{@monthly_snap_url}/#{@datetime})…"
-      es_api :put, "#{@monthly_snap_url}/#{@datetime}"
-
-      # Give the new backup time to show up
-      sleep 5
-
-      verify_create!
+      logger.info "Starting a new backup (#{backup_repo}/#{@snapshot_label})…"
+      r = es_api.snapshot.create repository: @backup_repo,
+                                 snapshot: @snapshot_label,
+                                 wait_for_completion: true
+      logger.info 'Snapshot complete. Time: ' \
+                  "#{r['snapshot']['duration_in_millis']}. " \
+                  "Results: #{r['snapshot']['shards'].inspect}"
+      fail "Snapshot failed! #{r.inspect}" if r['snapshot']['failures'].any?
     end
 
     def restore_test_index
       # Restore just the backup_test index to a new index
       logger.info 'Restoring the backup_test index…'
-      es_api(
-        :post,
-        "#{@monthly_snap_url}/#{@datetime}/_restore",
-        parameters: {
-          indices: @backup_index,
-          rename_pattern: @backup_index,
-          rename_replacement: @restore_index
-        }.to_json
-      )
-      verify_restored_index!
-    end
-
-    def index_shards(index)
-      r = es_api(:get, "#{@url}/#{index}/_status")
-      fail "Index #{index} not found" if r.code == 404
-      r.body.fetch('indices', {}).fetch(index, {})['shards']
-    end
-
-    def index_online?(index)
-      shards = index_shards(index)
-      shards && shards.select { |_k, v| v.find { |n| n['state'] != 'STARTED' } }.empty?
+      es_api.snapshot.restore repository: @backup_repo,
+                              snapshot: @snapshot_label,
+                              wait_for_completion: true,
+                              body: {
+                                indices: @backup_test_index,
+                                rename_pattern: @backup_test_index,
+                                rename_replacement: @restore_test_index
+                              }
     end
 
     def index_item(index, id)
-      es_api(:get, "#{@url}/#{index}/dummy/#{id}").body
-    end
-
-    def wait_for_index(index)
-      until index_online? index
-        logger.info 'Waiting for restored index to be available…'
-        sleep 1
-      end
+      es_api.get(index: index, type: 'dummy', id: id)['_source']['test_value']
     end
 
     def compare_index_item!(i)
-      # Loop until the restored version is available
-      until index_item(@restore_index, i)['found']
-        logger.info 'Waiting for restored index to be available…'
-        sleep 1
-      end
-
-      backup_item  = index_item(@backup_index, i)['_source']['test_value']
-      restore_item = index_item(@restore_index, i)['_source']['test_value']
+      backup_item  = index_item(@backup_test_index, i)
+      restore_item = index_item(@restore_test_index, i)
 
       (backup_item == restore_item) ||
         fail("Item #{i} in test restore doesn’t match.\n" \
@@ -270,45 +157,37 @@ module EverTools
              "Restored: #{restore_item}")
     end
 
-    def verify_restored_index!
-      # Compare each doc in the original backup_test index to the restored index
-
-      logger.info "Verifying the newly-restored #{@backup_index}…"
-      wait_for_index @restore_index
-
-      conf['test_size'].times { |i| compare_index_item! i }
-
-      logger.info 'Successfully verified the test data!'
+    def delete_test_indexes
+      [@restore_test_index, @backup_test_index].each do |test_index|
+        es_api.indices.delete index: test_index
+      end
     end
 
     # rubocop:disable Metrics/AbcSize, Lint/RescueException
     def run
       unless master?
-        logger.info 'This node is not the currently elected master, aborting ' \
-                    'backup.'
+        logger.info 'This node is not the currently elected master. Exiting.'
         exit 0
       end
 
-      configure_sentry!
-
-      # Remove the previous `restore_test` index, to avoid a race condition
-      # with checking the restored copy of this index
-      delete_index @restore_index if index? @restore_index
-
       insert_test_data
 
       # Create a new repo if none exists (typically at beginning of month)
-      create_repo unless repo?
-      make_new_backup
+      create_repo unless es_api.snapshot.get_repository[backup_repo]
+      create_snapshot
       restore_test_index
+      # Compare each doc in the original backup_test index to the restored index
+      logger.info "Verifying the newly-restored #{@backup_test_index}…"
+      test_size.times { |i| compare_index_item! i }
+      logger.info 'Successfully verified the test data!'
+      delete_test_indexes
 
       remove_expired_backups
       logger.info 'Finished'
     rescue Exception => e # Need to rescue "Exception" so that Sentry gets it
-      notify e if conf['env'] == 'prod'
+      notify e
       logger.fatal e.message
       logger.fatal e.backtrace
-      Raven.capture_exception(e)
       raise e
     end
     # rubocop:enable Metrics/AbcSize, Lint/RescueException

data/lib/elasticsearch_s3_backup/version.rb

@@ -1,5 +1,5 @@
 module EverTools
   class ElasticsearchS3Backup
-    VERSION = "1.0.1"
+    VERSION = '1.1.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch_s3_backup
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Eric Herot
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-08-24 00:00:00.000000000 Z
+date: 2015-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email:
 - eric.github@herot.com