es_dump_restore 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in es_dump_restore.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 PatientsLikeMe
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# es_dump_restore
+
+A utility for safely dumping the contents of an ElasticSearch index to a compressed file and restoring it
+later on.  This can be used for backups or for cloning an ElasticSearch index without needing to take down
+the server.
+
+The file format is a ZIP file containing the index metadata, the number of objects in the index, and a
+series of commands to be sent to the ElasticSearch bulk API.
+
+## Installation
+
+    gem install es_dump_restore
+
+## Usage
+
+To dump an ElasticSearch index to a file:
+
+    es_dump_restore dump ELASTIC_SEARCH_SERVER_URL INDEX_NAME DESTINATION_FILE
+
+To restore an index to an ElasticSearch server:
+
+    es_dump_restore restore ELASTIC_SEARCH_SERVER_URL DESTINATON_INDEX FILENAME
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/es_dump_restore

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'es_dump_restore'
+require 'es_dump_restore/app'
+
+EsDumpRestore::App.start

data/es_dump_restore.gemspec

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'es_dump_restore/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "es_dump_restore"
+  gem.version       = EsDumpRestore::VERSION
+  gem.authors       = ["Nat Budin"]
+  gem.email         = ["nbudin@patientslikeme.com"]
+  gem.description   = %q{A utility for dumping the contents of an ElasticSearch index to a compressed file and restoring the dumpfile back to an ElasticSearch server}
+  gem.summary       = %q{Dump ElasticSearch indexes to files and restore them back}
+  gem.homepage      = "https://github.com/patientslikeme/es_dump_restore"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'multi_json'
+  gem.add_dependency 'httpclient'
+  gem.add_dependency 'thor'
+  gem.add_dependency 'rubyzip'
+  gem.add_dependency 'progress_bar'
+end

data/lib/es_dump_restore/app.rb

@@ -0,0 +1,50 @@
+require "es_dump_restore/es_client"
+require "es_dump_restore/dumpfile"
+require "thor"
+require "progress_bar"
+
+module EsDumpRestore
+  class App < Thor
+
+    desc "dump URL INDEX_NAME FILENAME", "Creates a dumpfile based on the given ElasticSearch index"
+    def dump(url, index_name, filename)
+      client = EsClient.new(url, index_name)
+
+      Dumpfile.write(filename) do |dumpfile|
+        dumpfile.index = {
+          settings: client.settings,
+          mappings: client.mappings
+        }
+
+        client.start_scan do |scroll_id, total|
+          dumpfile.num_objects = total
+          bar = ProgressBar.new(total)
+
+          dumpfile.get_objects_output_stream do |out|
+            client.each_scroll_hit(scroll_id) do |hit|
+              metadata = { index: { _type: hit["_type"], _id: hit["_id"] } }
+              out.write("#{metadata.to_json}\n#{hit["_source"].to_json}\n")
+              bar.increment!
+            end
+          end
+        end
+      end
+    end
+
+    desc "restore URL INDEX_NAME FILENAME", "Restores a dumpfile into the given ElasticSearch index"
+    def restore(url, index_name, filename)
+      client = EsClient.new(url, index_name)
+
+      Dumpfile.read(filename) do |dumpfile|
+        client.create_index(dumpfile.index)
+
+        bar = ProgressBar.new(dumpfile.num_objects)
+        dumpfile.scan_objects(1000) do |batch, size|
+          client.bulk_index batch
+          bar.increment!(size)
+        end
+      end
+    end
+
+  end
+end

data/lib/es_dump_restore/dumpfile.rb

@@ -0,0 +1,80 @@
+require 'zip/zip'
+require 'multi_json'
+
+module EsDumpRestore
+  class Dumpfile < Zip::ZipFile
+    def self.write(filename, &block)
+      df = Dumpfile.new(filename, Zip::ZipFile::CREATE)
+      begin
+        yield df
+      ensure
+        df.close
+      end
+    end
+
+    def self.read(filename, &block)
+      df = Dumpfile.new(filename)
+      begin
+        yield df
+      ensure
+        df.close
+      end
+    end
+
+    def get_objects_input_stream(&block)
+      get_input_stream("objects", &block)
+    end
+
+    def get_objects_output_stream(&block)
+      get_output_stream("objects", nil, &block)
+    end
+
+    def num_objects
+      read_json_file("num_objects.json")["num_objects"]
+    end
+
+    def num_objects=(n)
+      write_json_file("num_objects.json", {num_objects: n})
+    end
+
+    def scan_objects(batch_size, &block)
+      get_objects_input_stream do |input|
+        loop do
+          commands = ""
+          items = 0
+
+          batch_size.times do
+            metadata = input.gets("\n")
+            break if metadata.nil?
+            commands << metadata
+
+            source = input.gets("\n")
+            commands << source
+
+            items += 1
+          end
+          break if commands.empty?
+
+          yield commands, items
+        end
+      end
+    end
+
+    def index=(index)
+      write_json_file("index.json", index)
+    end
+
+    def index
+      read_json_file("index.json")
+    end
+
+    private
+    def read_json_file(filename)
+      get_input_stream(filename) { |i| MultiJson.load(i.read) }
+    end
+
+    def write_json_file(filename, object)
+      get_output_stream(filename) { |o| o.write MultiJson.dump(object) }
+    end
+  end
+end

data/lib/es_dump_restore/es_client.rb

@@ -0,0 +1,62 @@
+require 'uri'
+require 'httpclient'
+require 'multi_json'
+
+module EsDumpRestore
+  class EsClient
+    attr_accessor :base_uri
+    attr_accessor :index_name
+
+    def initialize(base_uri, index_name)
+      @httpclient = HTTPClient.new
+      @index_name = index_name
+      @base_uri = URI.parse(base_uri + "/" + index_name + "/")
+    end
+
+    def mappings
+      request(:get, '_mapping')[index_name]
+    end
+
+    def settings
+      request(:get, '_settings')[index_name]
+    end
+
+    def start_scan(&block)
+      scroll = request(:get, '_search',
+        query: { search_type: 'scan', scroll: '10m', size: 500 },
+        body: MultiJson.dump({ query: { match_all: {} } }) )
+      total = scroll["hits"]["total"]
+      scroll_id = scroll["_scroll_id"]
+
+      yield scroll_id, total
+    end
+
+    def each_scroll_hit(scroll_id, &block)
+      loop do
+        batch = request(:get, '/_search/scroll', query: { scroll: '10m', scroll_id: scroll_id })
+        hits = batch["hits"]["hits"]
+        break if hits.empty?
+
+        hits.each do |hit|
+          yield hit
+        end
+      end
+    end
+
+    def create_index(metadata)
+      request(:post, "", :body => MultiJson.dump(metadata))
+    end
+
+    def bulk_index(data)
+      request(:post, "_bulk", :body => data)
+    end
+
+    private
+
+    def request(method, path, options={})
+      request_uri = @base_uri + path
+      response = @httpclient.request(method, request_uri, options)
+      MultiJson.load(response.content)
+    end
+  end
+end

data/lib/es_dump_restore/version.rb

@@ -0,0 +1,3 @@
+module EsDumpRestore
+  VERSION = "0.0.1"
+end

data/lib/es_dump_restore.rb

@@ -0,0 +1,3 @@
+require "es_dump_restore/version"
+require "es_dump_restore/es_client"
+require "es_dump_restore/app"

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: es_dump_restore
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Nat Budin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: progress_bar
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A utility for dumping the contents of an ElasticSearch index to a compressed
+  file and restoring the dumpfile back to an ElasticSearch server
+email:
+- nbudin@patientslikeme.com
+executables:
+- es_dump_restore
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/es_dump_restore
+- es_dump_restore.gemspec
+- lib/es_dump_restore.rb
+- lib/es_dump_restore/app.rb
+- lib/es_dump_restore/dumpfile.rb
+- lib/es_dump_restore/es_client.rb
+- lib/es_dump_restore/version.rb
+homepage: https://github.com/patientslikeme/es_dump_restore
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Dump ElasticSearch indexes to files and restore them back
+test_files: []
+has_rdoc: