es-dump 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODNmOTc0NGVhNmFkNmEzNDliODk5MjkxNTc0OTFlOTI2NjAyMGY2Yg==
+  data.tar.gz: !binary |-
+    YmQ2YjM3ZDk0NTY3YWRhMTRjYmQ2NWVkMTU3ODUzZTgyNWIxYjJhYQ==
+SHA512:
+  metadata.gz: !binary |-
+    Y2MxMWZkOWRlMjRhYWZjMGFhNzE0NTM0YWRjNTc4NzE0YzNlNDE4ODBlYTk5
+    NTAzZTU0Y2EwNDU5Yjk3MTY4ZmM2NTg5MjUzMjU2ZTA5MjQyN2FmODlhNTQ0
+    NDIyMDk4NmYxY2EwZDc5ZDVkYjE3Nzc4NGU4YTgwNWJkNDRjYWM=
+  data.tar.gz: !binary |-
+    ZmE3MTgyMTcyMGIxNzYyOTczOWU2YjJkNDUyMDBmMDNiN2RiYmY4ZjFlYjRm
+    ZTQ3MGI4MTUxYzM5MGExMmVjNTA1NjllYjIwNjE4YzJmZGE3YmMwMGVlMjJh
+    ZGY2YmViZTFiN2Y0MGM1MGYwNmI0ZjJlNDRlYjk0NDExZTM1NzE=

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm:
+- 2.0.0
+deploy:
+  provider: rubygems
+  api_key:
+    secure: Y60yEL3rsnFf/OkzLeIgI6MgEIunB1Rjb0p/Ig7L/mU3w8528R+PA1h0C1O6R0Fh3WfxkUoJ7gdEzS0D5wNlzUl4svhsrBfJlqYbxIjRr8TyEPDQC5M1K0rgSzio7rEKPYpg/G4aBkBtxKGmw9EIlPiGtVx06OxUKuma2+8+Fv8=
+  gem: es-dump
+  on:
+    repo: criteo/es-dump
+    all_branches: true
+    tags: true
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in es-dump.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Dan Forest Barbier
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,16 @@
+# ES-Dump
+
+A basic ElasticSearch dump/import tool
+
+## Usage
+
+Exporting entries
+
+    es-dump --host hostname --port port --index index_name --out index_backup.json
+
+Importing entries (will potentially overwrite existing values!)
+
+    es-dump --host hostname --port port --index index_name --in index_backup.json
+
+In case you have small (very large) entries, you can increase (decrease) the page size
+using the `--pagesize #ENTRIES` option.

data/Rakefile

@@ -0,0 +1,4 @@
+require "bundler/gem_tasks"
+
+task :default
+

data/bin/es-dump

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+require 'elasticsearch'
+require 'json'
+require 'optparse'
+
+options = {
+  :port => 9200,
+  :pagesize => 128,
+}
+
+$op = OptionParser.new
+$op.banner = 'Usage: es-dump [options]'
+$op.on('--host HOST', 'ES server hostname')                 {|v| options[:host] = v }
+$op.on('--port PORT', 'ES server port')                     {|v| options[:port] = v }
+$op.on('--index INDEX', 'Name of the index to export')      {|v| options[:index] = v }
+$op.on('--pagesize COUNT', 'Size of pages fetched from ES') {|v| options[:pagesize] = v.to_i }
+$op.on('--in INPUT', 'Input file for import mode')          {|v| options[:input] = v }
+$op.on('--out OUTPUT', 'Output file for export mode')       {|v| options[:output] = v }
+$op.on('-h', '--help', 'Prints this help message')          {|v| puts $op; exit }
+$op.parse!
+
+def error(message)
+  puts 'Error: ' + message
+  puts $op
+  exit
+end
+
+error 'You must specify a server hostname to connect to' if not options.has_key? :host
+error 'You must specify an index name' if not options.has_key? :index
+
+if not options.has_key? :output and not options.has_key? :input
+  error 'You must specify either an input or an output file'
+end
+
+puts "Connecting to ElasticSearch server at #{options[:host]}:#{options[:port]}"
+es = Elasticsearch::Client.new host: "#{options[:host]}:#{options[:port]}"
+offset = 0
+
+if options.has_key? :output
+  puts "Writing output file at #{options[:output]}"
+  output = File.new(options[:output], 'w')
+  loop do
+    begin
+      result = es.search index: options[:index],
+                         body: {
+                           from: offset,
+                           size: options[:pagesize],
+                         }
+    rescue Exception => e
+      error e.message
+    end
+
+    entries = result['hits']['hits']
+    for entry in entries
+      output.write JSON.pretty_generate(entry)
+      output.write "\n"
+    end
+
+    offset += entries.size
+    break if entries.size < options[:pagesize]
+  end
+
+  output.close
+elsif options.has_key? :input
+  puts "Reading input file at #{options[:input]}"
+  File.open(options[:input], 'r').each_line("}\n{") do |entry|
+    begin
+      entry = '{' + entry if entry[0] != '{' # Add missing { at the beginning
+      entry = entry[0..-2] # Remove extra { at the end
+      entry = JSON.parse entry
+
+      es.index index: options[:index],
+               type: entry['_type'],
+               id: entry['_id'],
+               body: entry['_source']
+    rescue Exception => e
+      error e.message
+    end
+
+    offset += 1
+  end
+end
+
+puts "Done, handled #{offset} entries"

data/es-dump.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "es-dump"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Dan Forest-Barbier"]
+  spec.email         = ["d.forestbarbier@criteo.com"]
+  spec.summary       = %q{A small tool for Elasticsearch backup and import}
+  spec.description   = %q{The data is dumped as JSON objects into the output file}
+  spec.homepage      = "https://github.com/criteo/es-dump"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: es-dump
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dan Forest-Barbier
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: The data is dumped as JSON objects into the output file
+email:
+- d.forestbarbier@criteo.com
+executables:
+- es-dump
+extensions: []
+extra_rdoc_files: []
+files:
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/es-dump
+- es-dump.gemspec
+homepage: https://github.com/criteo/es-dump
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A small tool for Elasticsearch backup and import
+test_files: []