elasticsearch_s3_backup 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1a2652e8865ae1bae268503eacb635d2ce586592
+  data.tar.gz: 11c55edfc00bdc8abb85aff511c939fe0eb4d9e6
+SHA512:
+  metadata.gz: e7e337aa678f99dba4ff872761039f15fa511c6d7246d955f562d52a46889e83bd0789ae15ef841419c6f2f8f424c0a757b1c829c0992683050fdf077a85f8fb
+  data.tar.gz: c27135eab4c5871cee1d48b0ed6f12f516040bf32049d6ac29dd017db3816223f14d1c91d4286052d03b68abfca4fc04ebe4fb259ad27737441396c024975234

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.2
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in elasticsearch_s3_backup.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Eric Herot
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# ElasticsearchS3Backup
+
+This gem creates a backup of your ElasticSearch cache and uploads it to S3.
+
+## Installation
+
+Install it using:
+
+    $ gem install elasticsearch_s3_backup
+
+## Configuration
+
+Create a YAML file called `/etc/s3_backup.yml` and give it the following contents:
+
+    ---
+    notification_email: @@NOTIFICATION_EMAIL_ADDRESS@@
+    test_size: 100
+    log: "/var/log/s3_backup/s3_backup.log"
+    new_repo_params:
+      bucket: @@BACKUPS_S3_BUCKET@@
+      max_snapshot_bytes_per_sec: 100mb
+      max_restore_bytes_per_sec: 500mb
+    env: stage
+    pagerduty_api_key: @@YOUR_PAGERDUTY_API_KEY@@
+    node_name: @@THIS_NODE'S_NAME@@
+    elasticsearch_auth_file: "/usr/local/elasticsearch/password"
+    cluster_name: @@YOUR_CLUSTER_NAME@@
+
+## Usage
+
+Just run the command:
+
+    $ es_s3_backup
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/evertrue/elasticsearch_s3_backup.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "elasticsearch_s3_backup"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/elasticsearch_s3_backup.gemspec

@@ -0,0 +1,40 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'elasticsearch_s3_backup/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'elasticsearch_s3_backup'
+  spec.version       = EverTools::ElasticsearchS3Backup::VERSION
+  spec.authors       = ['Eric Herot']
+  spec.email         = ['eric.github@herot.com']
+
+  spec.summary       = 'Backs up ElasticSearch to S3'
+  spec.description   = spec.description
+  spec.homepage      = 'https://nerds.evertrue.com'
+  spec.license       = 'MIT'
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+  else
+    fail 'RubyGems 2.0 or newer is required to protect against public gem ' \
+         'pushes.'
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'unirest'
+  spec.add_dependency 'faker'
+  spec.add_dependency 'pagerduty'
+  spec.add_dependency 'sentry-raven'
+end

data/exe/es_s3_backup

@@ -0,0 +1,4 @@
+require 'rubygems'
+require 'elasticsearch_s3_backup'
+
+EverTools::ElasticsearchS3Backup.new.run

data/lib/elasticsearch_s3_backup.rb

@@ -0,0 +1,316 @@
+require 'elasticsearch_s3_backup/version'
+require 'active_support/time'
+require 'unirest'
+require 'logger'
+require 'faker'
+require 'pagerduty'
+require 'yaml'
+require 'sentry-raven'
+
+module EverTools
+  class ElasticsearchS3Backup
+    def conf
+      @conf ||= YAML.load_file('/etc/s3_backup.yml')
+    end
+
+    def pagerduty
+      @pagerduty ||= Pagerduty.new conf['pagerduty_api_key']
+    end
+
+    def configure_sentry!
+      Raven.configure { config.dsn = conf['sentry_dsn'] }
+    end
+
+    def auth
+      @auth ||= File.read(conf['elasticsearch_auth_file']).strip.split ':'
+    end
+
+    def initialize
+      Unirest.default_header 'Accept', 'application/json'
+      Unirest.default_header 'Content-Type', 'application/json'
+      Unirest.timeout 30
+
+      @url = 'http://localhost:9200'
+
+      @backup_index  = 'backup_test'
+      @restore_index = 'restore_test'
+
+      now       = Time.new.utc
+      @monthly  = now.strftime '%m-%Y'
+      @datetime = now.strftime '%m-%d_%H%M'
+
+      @monthly_snap_url = "#{@url}/_snapshot/#{@monthly}"
+
+      @backup_timeout = 1.hours
+    end
+
+    def logger
+      # Set up logging
+      @logger ||= Logger.new(conf['log']).tap do |l|
+        l.level = Logger::INFO
+        l.progname = 's3_backup'
+        l.formatter =
+          proc do |severity, datetime, progname, msg|
+            "#{datetime.utc} [#{progname}] #{severity}: #{msg}\n"
+          end
+      end
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def es_api(method, uri, params = {})
+      tries = 3
+      begin
+        r = Unirest.send(method, uri, params)
+        case r.code
+        when 200..299
+          return r
+        when 400..499
+          logger.debug "#{method} request to #{uri} received #{r.code} (params: #{params.inspect})\n" \
+                      "Body:\n" \
+                      "#{r.body}\n"
+          return r
+        end
+        # byebug
+        fail "#{method.upcase} request to #{uri} failed (params: #{params.inspect})\n" \
+             "Response code: #{r.code}\n" \
+             "Body:\n" \
+             "#{r.body}\n"
+      rescue RuntimeError => e
+        tries -= 1
+        retry if e.message == 'Request Timeout' && tries > 0
+        raise e
+      end
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    def master?
+      es_api(:get, "#{@url}/_cat/master").body[0]['node'] == conf['node_name']
+    end
+
+    # Check if an index exists
+    def index?(uri)
+      es_api(:get, "#{@url}/#{uri}").code == 200
+    end
+
+    # Check if a backup repo exists
+    def repo?
+      es_api(:get, @monthly_snap_url).code == 200
+    end
+
+    def notify(e)
+      pagerduty.trigger(
+        'prod Elasticsearch S3 failed',
+        client: conf['node_name'],
+        details: "#{e.message}\n\n#{e.backtrace}"
+      )
+    end
+
+    def insert_test_data
+      # Generate some test data using Faker
+      #
+      # Uses Bitcoin addresses for their random, hash-like nature
+      # Creates the `backup_test` index if necessary
+      # Updates the set of `dummy` documents in the `backup_test` on every run
+
+      logger.info 'Generating test data using Faker…'
+      conf['test_size'].times do |i|
+        es_api(
+          :put,
+          "#{@url}/#{@backup_index}/dummy/#{i}",
+          parameters: {
+            test_value: Faker::Bitcoin.address
+          }.to_json
+        )
+      end
+    end
+
+    def delete_index(index)
+      logger.info "Deleting index: #{index}"
+      es_api(
+        :delete,
+        "#{@url}/#{index}",
+        auth: { user: auth.first, password: auth.last }
+      )
+    end
+
+    def create_repo
+      new_repo_params = conf['new_repo_params'].merge(
+        type: 's3',
+        settings: {
+          base_path: "/elasticsearch/#{cluster_name}/#{conf['env']}/#{@monthly}",
+          server_side_encryption: true
+        }
+      )
+
+      logger.info 'Creating a new monthly ES backup repo…'
+      es_api(
+        :put,
+        @monthly_snap_url,
+        parameters: new_repo_params.to_json
+      )
+    end
+
+    def valid_date?(date)
+      Time.strptime(date, '%m-%Y') rescue false
+    end
+
+    def dated_repos
+      es_api(:get, "#{@url}/_snapshot").body.keys.select { |r| valid_date? r }
+    end
+
+    def remove_expired_backups
+      # Remove 3 month old repos
+      logger.info "Removing backups older than #{3.months.ago.strftime '%m-%Y'}"
+      dated_repos.select { |b| Time.strptime(b, '%m-%Y') < 3.months.ago }.each do |repo|
+        logger.info "Removing #{repo}"
+        es_api :delete, "#{@url}/_snapshot/#{repo}"
+      end
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def snapshot
+      backup_uri = "#{@monthly_snap_url}/#{@datetime}"
+      status_req = es_api :get, "#{backup_uri}/_status"
+      if status_req.code == 404 ||
+         status_req.body['snapshots'].empty?
+        fail "Could not find the backup I just created (#{backup_uri})"
+      end
+      snapshot = status_req.body['snapshots'].first
+      logger.info "Backup state: #{snapshot['state']} " \
+                  "(finished shard #{snapshot['shards_stats']['done']} of " \
+                  "#{snapshot['shards_stats']['total']})"
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    def backup_complete?
+      case snapshot['state']
+      when 'IN_PROGRESS', 'STARTED'
+        return false
+      when 'SUCCESS'
+        return true
+      end
+      fail "Backup failed!\n" \
+           "State: #{snapshot['state']}\n" \
+           "Response Body: #{status_req.body}"
+    end
+
+    def verify_create!
+      # Check the status of the backup for up to an hour
+      backup_start_time = Time.now.utc
+      until Time.now.utc > (backup_start_time + @backup_timeout)
+        return true if backup_complete?
+        # Don't hammer the status endpoint
+        sleep 15
+      end
+
+      fail 'Create timed out'
+    end
+
+    def make_new_backup
+      # Make a backup (full on new month, incremental otherwise)
+      logger.info "Starting a new backup (#{@monthly_snap_url}/#{@datetime})…"
+      es_api :put, "#{@monthly_snap_url}/#{@datetime}"
+
+      # Give the new backup time to show up
+      sleep 5
+
+      verify_create!
+    end
+
+    def restore_test_index
+      # Restore just the backup_test index to a new index
+      logger.info 'Restoring the backup_test index…'
+      es_api(
+        :post,
+        "#{@monthly_snap_url}/#{@datetime}/_restore",
+        parameters: {
+          indices: @backup_index,
+          rename_pattern: @backup_index,
+          rename_replacement: @restore_index
+        }.to_json
+      )
+      verify_restored_index!
+    end
+
+    def index_shards(index)
+      r = es_api(:get, "#{@url}/#{index}/_status")
+      fail "Index #{index} not found" if r.code == 404
+      r.body.fetch('indices', {}).fetch(index, {})['shards']
+    end
+
+    def index_online?(index)
+      shards = index_shards(index)
+      shards && shards.select { |_k, v| v.find { |n| n['state'] != 'STARTED' } }.empty?
+    end
+
+    def index_item(index, id)
+      es_api(:get, "#{@url}/#{index}/dummy/#{id}").body
+    end
+
+    def wait_for_index(index)
+      until index_online? index
+        logger.info 'Waiting for restored index to be available…'
+        sleep 1
+      end
+    end
+
+    def compare_index_item!(i)
+      # Loop until the restored version is available
+      until index_item(@restore_index, i)['found']
+        logger.info 'Waiting for restored index to be available…'
+        sleep 1
+      end
+
+      backup_item  = index_item(@backup_index, i)['_source']['test_value']
+      restore_item = index_item(@restore_index, i)['_source']['test_value']
+
+      (backup_item == restore_item) ||
+        fail("Item #{i} in test restore doesn’t match.\n" \
+             "Original: #{backup_item}\n" \
+             "Restored: #{restore_item}")
+    end
+
+    def verify_restored_index!
+      # Compare each doc in the original backup_test index to the restored index
+
+      logger.info "Verifying the newly-restored #{@backup_index}…"
+      wait_for_index @restore_index
+
+      conf['test_size'].times { |i| compare_index_item! i }
+
+      logger.info 'Successfully verified the test data!'
+    end
+
+    # rubocop:disable Metrics/AbcSize, Lint/RescueException
+    def run
+      unless master?
+        logger.info 'This node is not the currently elected master, aborting ' \
+                    'backup.'
+        exit 0
+      end
+
+      configure_sentry!
+
+      # Remove the previous `restore_test` index, to avoid a race condition
+      # with checking the restored copy of this index
+      delete_index @restore_index if index? @restore_index
+
+      insert_test_data
+
+      # Create a new repo if none exists (typically at beginning of month)
+      create_repo unless repo?
+      make_new_backup
+      restore_test_index
+
+      remove_expired_backups
+      logger.info 'Finished'
+    rescue Exception => e # Need to rescue "Exception" so that Sentry gets it
+      notify e if conf['env'] == 'prod'
+      logger.fatal e.message
+      logger.fatal e.backtrace
+      Raven.capture_exception(e)
+      raise e
+    end
+    # rubocop:enable Metrics/AbcSize, Lint/RescueException
+  end
+end

data/lib/elasticsearch_s3_backup/version.rb

@@ -0,0 +1,5 @@
+module EverTools
+  class ElasticsearchS3Backup
+    VERSION = "1.0.1"
+  end
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: elasticsearch_s3_backup
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Eric Herot
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-08-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: unirest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pagerduty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sentry-raven
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- eric.github@herot.com
+executables:
+- es_s3_backup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- elasticsearch_s3_backup.gemspec
+- exe/es_s3_backup
+- lib/elasticsearch_s3_backup.rb
+- lib/elasticsearch_s3_backup/version.rb
+homepage: https://nerds.evertrue.com
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Backs up ElasticSearch to S3
+test_files: []
+has_rdoc: