elasticsearch-snapshot 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3baa2e277201c3605f2acd8b2a91aa8f56cf6028
+  data.tar.gz: e7bc8ca73a82312a14f5e5aeb84c9c37bff0484f
+SHA512:
+  metadata.gz: 6d82001e9b15ecc72efd886e30336856315b7afb3db50430795cd7e392e2908017a24c10710ba9b100b0a48d9204b51ae8f7997b4ca052cead70d83d2ab913b8
+  data.tar.gz: 25bac3a8f686bcc85904ca343b23ea4cafb1f3900ea0ce3dfc363a7c94288509048a80a7977adb09b5ea1aba156b92b0be1938091ded09f1322e31fca3137ba6

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.3
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in elasticsearch-snapshot.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# Elasticsearch::Snapshot
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/elasticsearch/snapshot`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'elasticsearch-snapshot'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install elasticsearch-snapshot
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake false` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/elasticsearch-snapshot.
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "elasticsearch/snapshot"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/elasticsearch-snapshot.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'elasticsearch/snapshot/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "elasticsearch-snapshot"
+  spec.version       = Elasticsearch::Snapshot::VERSION
+  spec.authors       = ["Vlad Gorodetsky"]
+  spec.email         = ["v@gor.io"]
+
+  spec.summary       = %q{Snapshots, restores, and migrates ElasticSearch indices.}
+  spec.description   = %q{Snapshots, restores, and migrates ElasticSearch indices between clusters.}
+  spec.homepage      = "https://github.com/bai/elasticsearch-snapshot"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "thor", "~> 0.19.1"
+end

data/lib/elasticsearch/snapshot/migrate.rb

@@ -0,0 +1,50 @@
+module Elasticsearch
+  module Snapshot
+    class Migrate < Task
+      attr_reader :source, :target, :repo
+
+      def initialize(source, target, repo)
+        @source = smart_add_url_protocol(source)
+        @target = smart_add_url_protocol(target)
+        @repo = repo
+      end
+
+      def perform
+        begin
+          list_indexes_to_backup = JSON(open(command_list_indexes_to_backup(source)).read)['indices'].keys.delete_if { |i| i =~ NON_RESTORABLE_INDICES_REGEX }.join(',')
+        rescue
+          puts "Unable to get indices from ElasticSearch."
+          exit(1)
+        end
+
+        begin
+          snapshots = JSON(open(command_get_last_successful_snapshot(source, repo)).read)
+        rescue
+          puts "Unable to get list of snapshots from ElasticSearch. Does `#{repo}` repository exist?"
+          exit(1)
+        end
+
+        if snapshots["snapshots"].last["state"] == "SUCCESS"
+          begin
+            snapshot_pipe = IO.popen command_close_indices(target, list_indexes_to_backup)
+            Process.waitall
+          rescue
+            puts "Unable to close indices"
+            exit(1)
+          end
+
+          snapshot_date = snapshots["snapshots"].last["snapshot"]
+          puts "Restoring from snapshot #{snapshot_date}"
+          snapshot_pipe = IO.popen command_restore_snapshot(target, repo, snapshot_date, list_indexes_to_backup)
+          Process.waitall
+
+          restore_result = snapshot_pipe.readlines
+          puts restore_result
+        else
+          puts "Last snapshot was not successful."
+          exit(1)
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/snapshot/purge.rb

@@ -0,0 +1,36 @@
+module Elasticsearch
+  module Snapshot
+    class Purge < Task
+      attr_reader :cluster, :repo, :keep
+
+      def initialize(cluster, repo, keep = 30)
+        @cluster = cluster
+        @repo = repo
+        @keep = keep
+      end
+
+      def perform
+        raise unless keep > 0
+
+        snapshot_pipe = IO.popen command_list_snapshots(cluster, repo)
+        Process.waitall
+
+        result = snapshot_pipe.readlines
+        raise if result.length > 1
+
+        result = JSON.load result.first
+        raise unless result['snapshots']
+
+        result['snapshots'][0...-keep].each do |s|
+          snapshot_to_delete = command_delete_a_snapshot(cluster, repo) + "/#{s['snapshot']}"
+          pipe = IO.popen snapshot_to_delete
+          Process.waitall
+          result = JSON.load(pipe.readlines.first)
+          info("Purged snapshot #{s['snapshot']}") if result['acknowledged'] == true
+        end
+      rescue
+        error "Cleanup aborted: #{$!}"
+      end
+    end
+  end
+end

data/lib/elasticsearch/snapshot/snapshot.rb

@@ -0,0 +1,39 @@
+module Elasticsearch
+  module Snapshot
+    class Snapshot < Task
+      attr_reader :cluster, :repo
+
+      def initialize(cluster, repo)
+        @cluster = cluster
+        @repo = repo
+      end
+
+      def perform
+        list_indexes_to_backup = JSON(open(command_list_indexes_to_backup(cluster)).read)['indices'].keys.delete_if { |i| i =~ NON_RESTORABLE_INDICES_REGEX }
+
+        info "Snapshotting indexes: #{list_indexes_to_backup.join(', ')}"
+
+        snapshot_pipe = IO.popen command_take_a_snapshot(cluster, repo) + %Q( -d '{"indices": "#{list_indexes_to_backup.join(',')}","include_global_state":false}')
+        Process.waitall
+
+        snapshot_result = snapshot_pipe.readlines
+        raise if snapshot_result.length > 1
+
+        snapshot_result = JSON.load snapshot_result.first
+
+        if snapshot_result.keys.include? 'error'
+          error snapshot_result['error']
+        else
+          raise unless snapshot_result['snapshot']
+          if snapshot_result['snapshot']['state'] == 'SUCCESS'
+            info "Snapshot took #{snapshot_result['snapshot']['duration_in_millis']} ms"
+          else
+            error "Snapshot failed: #{snapshot_result['failures'].join(' ')}"
+          end
+        end
+      rescue
+        error "Snapshot aborted: #{$!}"
+      end
+    end
+  end
+end

data/lib/elasticsearch/snapshot/task.rb

@@ -0,0 +1,53 @@
+module Elasticsearch
+  module Snapshot
+    class Task
+      NON_RESTORABLE_INDICES_REGEX = /marvel|kibana/
+
+      protected
+
+      def command_list_indexes_to_backup(cluster)
+        %Q(#{cluster}/_stats/indices)
+      end
+
+      def command_take_a_snapshot(cluster, repo)
+        %Q(curl -s -X PUT #{cluster}/_snapshot/#{repo}/$(date +"%Y%m%d-%H%M%S")?wait_for_completion=true)
+      end
+
+      def command_list_snapshots(cluster, repo)
+        %Q(curl -s #{cluster}/_snapshot/#{repo}/_all)
+      end
+
+      def command_delete_a_snapshot(cluster, repo)
+        %Q(curl -s -X DELETE #{cluster}/_snapshot/#{repo})
+      end
+
+      def command_get_last_successful_snapshot(cluster, repo)
+        %Q(#{cluster}/_snapshot/#{repo}/_all?pretty=true)
+      end
+
+      def command_close_indices(cluster, indices)
+        %Q(curl -XPOST #{cluster}/#{indices}/_close)
+      end
+
+      def command_restore_snapshot(cluster, repo, snapshot_name, indices)
+        %Q(curl -XPOST #{cluster}/_snapshot/#{repo}/#{snapshot_name}/_restore -d '{"partial":true,"indices":"#{indices}","include_global_state":false}')
+      end
+
+      def info(msg)
+        puts '[INFO]' + msg
+      end
+
+      def error(msg)
+        puts '[ERROR]' + msg
+      end
+
+      def smart_add_url_protocol(uri)
+        if uri[/\Ahttp:\/\//] || uri[/\Ahttps:\/\//]
+          uri
+        else
+          "http://#{uri}"
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/snapshot/version.rb

@@ -0,0 +1,5 @@
+module Elasticsearch
+  module Snapshot
+    VERSION = "0.1.0"
+  end
+end

data/lib/elasticsearch/snapshot.rb

@@ -0,0 +1,12 @@
+require 'open-uri'
+
+require "elasticsearch/snapshot/version"
+require "elasticsearch/snapshot/task"
+require "elasticsearch/snapshot/snapshot"
+require "elasticsearch/snapshot/purge"
+require "elasticsearch/snapshot/migrate"
+
+module Elasticsearch
+  module Snapshot
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: elasticsearch-snapshot
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vlad Gorodetsky
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-09-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+description: Snapshots, restores, and migrates ElasticSearch indices between clusters.
+email:
+- v@gor.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- elasticsearch-snapshot.gemspec
+- lib/elasticsearch/snapshot.rb
+- lib/elasticsearch/snapshot/migrate.rb
+- lib/elasticsearch/snapshot/purge.rb
+- lib/elasticsearch/snapshot/snapshot.rb
+- lib/elasticsearch/snapshot/task.rb
+- lib/elasticsearch/snapshot/version.rb
+homepage: https://github.com/bai/elasticsearch-snapshot
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Snapshots, restores, and migrates ElasticSearch indices.
+test_files: []