elastic-util 0.1.3 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 86b2acc63f7a0ca1b6b9664c0f88f124e87e93cd
-  data.tar.gz: c96da7fa8d85ba9c2c3555d61165e4804ef6b6ea
+SHA256:
+  metadata.gz: b30dc1de09489fede6a09227ff9a8a292f416fdcf8e9eb041bcb46c08cbd6767
+  data.tar.gz: 5b604aafcae78e703530acec175bf1235178cb784711c2bcb8a665b9849c7792
 SHA512:
-  metadata.gz: f34401fb4c4e3b103a1943fc6a5911a0ae8f6289679c86c403c4737fb60c62501cfc7d563b92ec44dc2d8be48136e8a2f35550f53f0cc5928e3b9097827d95b9
-  data.tar.gz: ca82e26dc8f73e6ef7437655ded818727a65934797b94f79500db39021ed453362f5fa793ecb4c607721e30a173d5bb367e2311239f9dd53e2eb56de04862c23
+  metadata.gz: 4d4f94be2ca1d450ea260ab4d6f8450e8dfb8025f52f72281a932dc33480fdc0a965175166d9aa50000d386a59c8b65347b02aac6472e68432f663798f2d2e0b
+  data.tar.gz: '0029396b7a2bc8960f10532a8bdec303e4e3e813c8acbbab795c4dffbaf0e36cb707cf311acd363a5c34237b3ce7ea0454a37c1d261d491cc6b6919a64c6da8e'

data/HISTORY.md

@@ -0,0 +1,14 @@
+# History
+
+This is a changelog for the changes made to the `elastic-util` gem.
+
+### v0.1.8
+* improved `--help` output and error messages
+
+### v0.1.7
+* basic auth support inside the url argument using the format username:password@url
+* https support
+
+### v0.1.6
+* New backup option --replace-types
+

data/bin/elastic-util

@@ -3,17 +3,25 @@ require 'elastic-util'
 require 'optparse'
 
 prog_name = "elastic-util" # $0.split('/').last
-usage = "Usage: #{prog_name} [backup|restore] [options]"
+prog_usage = "usage: #{prog_name} [backup|restore] [url] [directory] [options]"
 args = ARGV.dup
 command_name = args.shift
 
 case command_name
 
 when "backup"
-
+  banner = <<-EOT
+usage: #{prog_name} #{command_name} [url] [directory] [options]
+EOT
+  footer = <<-EOT
+The backup command provides way to dump Elasticsearch index data.
+The /_search API is used to paginate requests for each index being exported.
+The data is written to specified directory, with a file for each api request made.
+Example: #{prog_name} #{command_name} http://localhost:9300 /tmp/local-elastic-data
+EOT
   options = {}
   optparse = OptionParser.new do |opts|
-    opts.banner = "Usage: #{prog_name} #{command_name} [url] [directory] [options]"
+    opts.banner = banner
     opts.on('--indices x,y,z', Array, "The indices to backup. Default is all.") do |val|
       options[:indices] = val.collect {|it| it.strip }
     end
@@ -23,6 +31,15 @@ when "backup"
     opts.on('--exclude-fields x,y,z', Array, "The fields to exclude from backup. Default is '_id'.") do |val|
       options[:exclude_fields] = val.collect {|it| it.strip }
     end
+    opts.on('--replace-types type1:_doc,type2:_doc', Array, "Replace certain types with a different type.") do |val|
+      options[:replace_types] = {}
+      val.each  do |it| 
+        pair = it.split(":").collect {|p| p.strip }
+        if pair.size == 2
+          options[:replace_types][pair[0]] = pair[1]
+        end
+      end
+    end
     opts.on( '-s', '--size NUMBER', "The size api parameter. This dicates the size of the files and api payloads. Default is 1000." ) do |val|
       options[:size] = val.to_i
     end
@@ -35,27 +52,24 @@ when "backup"
     opts.on( '-q', '--quiet', "Don't print to stdout. Default is false." ) do |val|
       options[:quiet] = true
     end
+    opts.on( '-d', '--dry-run', "Don't actually backup data, just print what would have happened. Default is false." ) do |val|
+      options[:dry] = true
+    end
     opts.on('-h', '--help', "Prints this help" ) do
-      puts opts
-      exit
+      puts opts, footer
+      exit 0
     end
   end
   optparse.parse!(args)
-  url = args[0]
-  backup_directory = args[1]
-  if url.nil? || url.empty?
-    $stderr.puts "#{prog_name}: missing required argument [url]"
-    $stderr.puts optparse
-    exit 1
-  end
-  if backup_directory.nil? || backup_directory.empty?
-    $stderr.puts "#{prog_name}: missing required argument [directory]"
-    $stderr.puts optparse
+  if args.count != 2
+    $stderr.puts "#{prog_name}: wrong number of arguments. Expected 2 and got #{args.count}: #{args.join(', ')}"
+    # $stderr.puts optparse
+    $stderr.puts banner
+    $stderr.puts "See `#{prog_name} #{command_name} --help` for more usage information."
     exit 1
   end
-  
   begin
-    result = ElasticUtil.backup(url, backup_directory, options)
+    result = ElasticUtil.backup(args[0], args[1], options)
     exit 0
   rescue ElasticUtil::Error => err
     $stderr.puts "#{prog_name}: #{err.message}"
@@ -65,34 +79,39 @@ when "backup"
 
 
 when "restore"
-
+  banner = <<-EOT
+usage: #{prog_name} #{command_name} [url] [directory] [options]
+EOT
+  footer = <<-EOT
+The restore command provides way to restore Elasticsearch index data.
+The /_bulk API is used to upload index data from the specified directory.
+The directory should contain data generated by the `#{prog_name} backup` command.
+e.g. #{prog_name} #{command_name} http://localhost:9400 /tmp/local-elastic-data
+EOT
   options = {}
   optparse = OptionParser.new do |opts|
-    opts.banner = "Usage: #{prog_name} #{command_name} [url] [directory] [options]"
+    opts.banner = banner
     opts.on( '-q', '--quiet', "Don't print to stdout. Default is false." ) do |val|
       options[:quiet] = true
     end
+    opts.on( '-d', '--dry-run', "Don't actually restore data, just print what would have happened. Default is false." ) do |val|
+      options[:dry] = true
+    end
     opts.on('-h', '--help', "Prints this help" ) do
-      puts opts
-      exit
+      puts opts, footer
+      exit 0
     end
   end
   optparse.parse!(args)
-  url = args[0]
-  backup_directory = args[1]
-  if url.nil? || url.empty?
-    $stderr.puts "#{prog_name}: missing required argument [url]"
-    $stderr.puts optparse
-    exit 1
-  end
-  if backup_directory.nil? || backup_directory.empty?
-    $stderr.puts "#{prog_name}: missing required argument [directory]"
-    $stderr.puts optparse
+  if args.count != 2
+    $stderr.puts "#{prog_name}: wrong number of arguments. Expected 2 and got #{args.count}: #{args.join(', ')}"
+    # $stderr.puts optparse
+    $stderr.puts banner
+    $stderr.puts "See `#{prog_name} #{command_name} --help` for more usage information."
     exit 1
   end
-  
   begin
-    result = ElasticUtil.restore(url, backup_directory, options)
+    result = ElasticUtil.restore(args[0], args[1], options)
     exit 0
   rescue ElasticUtil::Error => err
     $stderr.puts "#{prog_name}: #{err.message}"
@@ -100,10 +119,28 @@ when "restore"
     exit 1
   end
 
-when "-v"
+when "-h", "--help"
+  puts prog_usage
+  puts <<-EOT
+The elastic-util command provides way to dump and restore Elasticsearch index data.
+Example: #{prog_name} backup  http://localhost:9300 /tmp/local-elastic-data
+         #{prog_name} restore http://localhost:9400 /tmp/local-elastic-data
+EOT
+  exit 0
+when "-v","--version","version"
   puts ElasticUtil::VERSION
-else
-  $stderr.puts usage
+when "", nil
+  $stderr.puts "#{prog_name}: missing required argument [backup|restore]"
+  # $stderr.puts "Commands:"
+  # $stderr.puts "\tbackup"
+  # $stderr.puts "\trestore"
+  $stderr.puts prog_usage
+  $stderr.puts "See `#{prog_name} #{command_name} --help` for more usage information."
   exit 1
+else
+  # $stderr.puts "#{prog_name}: '#{command_name}' is not a recognized elastic-util command. See `#{prog_name} --help`"
+  $stderr.puts "#{prog_name}: '#{command_name}' is not a recognized elastic-util command."
+  $stderr.puts prog_usage
+  exit 3
 end
 

data/elastic-util.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["James Dickson"]
   spec.email         = ["dickson.james@gmail.com"]
   spec.summary       = "Provides backup and restore for ElasticSearch data"
-  spec.description   = "ElasticUtil uses ElasticSearch's scroll and _bulk APIs to dump and restore indices"
+  spec.description   = "ElasticUtil uses ElasticSearch's /_search and /_bulk APIs to dump and restore indices"
   #spec.homepage      = "http://www.elastic-util.com"
   spec.license       = "MIT"
 

data/lib/elastic_util.rb

@@ -1,18 +1,19 @@
 # encoding: utf-8
 require 'net/http'
+require 'openssl'
 require 'json'
 require 'fileutils'
 
-# This module provides a way to backup and restore elasticsearch data.
+# This module provides a way to backup and restore Elasticsearch data.
 #
 # @example Backup data from one elasticsearch cluster and restore it to another.
 #
-#     ElasticUtil.backup('http://localhost:9200', '/tmp/mybackup', {size:5000})
-#     ElasticUtil.restore('http://localhost:9201', '/tmp/mybackup')
+#     ElasticUtil.backup('http://localhost:9300', '/tmp/local-elastic-data', {size:5000})
+#     ElasticUtil.restore('http://localhost:9301', '/tmp/local-elastic-data')
 #
 module ElasticUtil
 
-  VERSION = "0.1.3"
+  VERSION = "0.1.8"
 
   # The name of the data directory, relative to the user provided backup directory.
   DUMP_DIR = "es_data"
@@ -20,29 +21,29 @@ module ElasticUtil
   # A class to be raised for any known error condition.
   class Error < StandardError; end
 
-  # Backup elasticsearch data to a local directory.
+  # Backup Elasticsearch data to a local directory.
   #
   # This uses ElasticSearch's scroll api to fetch all records for indices
   # and write the data to a local directory. The files it generates are given a 
   # .json.data extension. They are not valid JSON files, but rather are in the 
   # format expected by ElasticSearch's _bulk api.
   #
-  # So #restore simply has to POST the contents of each file.
+  # So #restore simply has to POST the contents of each file as Content-Type: application/x-ndjson
   #
-  # Use the :size option to change the number or results to fetch at once,
-  # and also the size of the data files generated.
-  # The latter correlates to the of the the api requests made in #restore.
+  # Use the :size option to change the number of results to fetch at once and also the size of the data files generated.
+  # Increasing size means larger files and fewer api requests for both #backup and the subsequent #restore of that data.
   #
-  # @example Backup default elasticsearch running locally.
+  # @example Backup default Elasticsearch running locally.
   #
-  #     ElasticUtil.backup('http://localhost:9200', '/tmp/mybackup')
+  #     ElasticUtil.backup('http://localhost:9300', '/tmp/local-elastic-data')
   #
-  # @param [String] url The url of the elasticsearch cluster eg. 'http://localhost:9200'
+  # @param [String] url The url of the Elasticsearch cluster eg. 'http://localhost:9300'
   # @param [String] backup_dir The local directory to store data in. eg. '/tmp/es2.4'
   # @param [Hash] opts The options for this backup.
   # @option opts [Array] :indices The indices to backup. Default is all.
   # @option opts [Array] :exclude_indices Exclude certain indexes.
   # @option opts [Array] :exclude_fields Exclude certain fields. Default is ['_id'].
+  # @option opts [Array] :replace_types Replace certain types with a different type, separated by a colon. eg. 'type1:type2' or 'stat:_doc'
   # @option opts [String] :scroll The scroll api parameter, Default is '5m'.
   # @option opts [Integer] :size The size api parameter. Default is 1000.
   # @option opts [true] :force Delete existing backup directory instead of erroring. Default is false.
@@ -52,29 +53,32 @@ module ElasticUtil
   #
   def self.backup(url, backup_dir, opts={})
     start_time = Time.now
+    url = url.strip.chomp("/")
     backup_dir = backup_dir.strip
     path = File.join(backup_dir.strip, DUMP_DIR)
     indices = []
-    
+
+    if opts[:dry]
+      puts "(DRY RUN) Started backup" unless opts[:quiet]
+    else
+      puts "Started backup" unless opts[:quiet]
+    end
+
     # ping it first
+    response = nil
     uri = URI(url)
-    response = Net::HTTP.get_response(uri)
-    http = Net::HTTP.new(uri.host, uri.port)
-    http.read_timeout = 5
-    http.open_timeout = 5
-    response = http.start() {|http|
-      http.get("/")
-    }
+    response = api_get(uri)
     if !response.is_a?(Net::HTTPSuccess)
-      raise Error, "Unable to reach elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
+      raise Error, "Unable to reach Elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
     end
 
     # determine indices to backup, default is everything.
     if opts[:indices]
       indices = opts[:indices]
     else
+      response = nil
       uri = URI(url + "/_cat/indices?format=json")
-      response = Net::HTTP.get_response(uri)
+      response = api_get(uri)
       if !response.is_a?(Net::HTTPSuccess)
         raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
       end
@@ -109,6 +113,14 @@ module ElasticUtil
     end
     FileUtils.mkdir_p(path)
 
+    if opts[:dry]
+      indices.each_with_index do |index_name, i|
+        puts "(#{i+1}/#{indices.size}) backing up index #{index_name}" unless opts[:quiet]
+      end
+      puts "(DRY RUN) Finished backup of Elasticsearch #{url} to directory #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
+      return 0
+    end
+
     # dump data
     indices.each_with_index do |index_name, i|
       puts "(#{i+1}/#{indices.size}) backing up index #{index_name}" unless opts[:quiet]
@@ -123,7 +135,7 @@ module ElasticUtil
       }
       uri.query = URI.encode_www_form(params)
       # puts "HTTP REQUEST #{uri.inspect}"
-      response = Net::HTTP.get_response(uri)
+      response = api_get(uri)
       if !response.is_a?(Net::HTTPSuccess)
         raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
       end
@@ -131,7 +143,7 @@ module ElasticUtil
       raise Error, "No scroll_id returned in response:\n#{response.inspect}" unless json_response['_scroll_id']
       scroll_id = json_response['_scroll_id']
       hits = json_response['hits']['hits']
-      save_bulk_data(path, hits)
+      save_bulk_data(path, hits, nil, opts)
       
       file_index = 1
       # scroll requests
@@ -143,7 +155,7 @@ module ElasticUtil
         }
         uri.query = URI.encode_www_form(params)
         # puts "HTTP REQUEST #{uri.inspect}"
-        response = Net::HTTP.get_response(uri)
+        response = api_get(uri)
         if !response.is_a?(Net::HTTPSuccess)
           raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
         end
@@ -151,27 +163,23 @@ module ElasticUtil
         raise Error, "No scroll_id returned in response:\n#{response.inspect}\n#{response.body.to_s}" unless json_response['_scroll_id']
         scroll_id = json_response['_scroll_id']
         hits = json_response['hits']['hits']
-        if file_index > 0
-          save_bulk_data(path, hits, file_index)
-        else
-          save_bulk_data(path, hits)
-        end
+        save_bulk_data(path, hits, file_index, opts)
         file_index += 1
       end
     end
     
-    puts "Finished backup of elasticsearch #{url} to directory #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
+    puts "Finished backup of Elasticsearch #{url} to directory #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
     return true
   end
 
-  # Restore elasticsearch data from a backup.
+  # Restore Elasticsearch data from a backup.
   # This will do a POST to the _bulk api for each file in the backup directory.
   #
   # @example Restore local cluster with our backup.
   #
-  #     ElasticUtil.restore('http://localhost:9201', '/tmp/mybackup')
+  #     ElasticUtil.restore('http://localhost:9301', '/tmp/local-elastic-data')
   #
-  # @param [String] url The url of the elasticsearch cluster eg. 'http://localhost:9200'.
+  # @param [String] url The url of the Elasticsearch cluster eg. 'http://localhost:9200'.
   # @param [String] backup_dir The backup directory.
   # @param [Hash] opts The options for this backup.
   # @option opts [true] :quiet Don't print anything. Default is false.
@@ -180,9 +188,16 @@ module ElasticUtil
   #
   def self.restore(url, backup_dir, opts={})
     start_time = Time.now
+    url = url.strip.chomp("/")
     backup_dir = backup_dir.strip
     path = File.join(backup_dir.strip, DUMP_DIR)
 
+    if opts[:dry]
+      puts "(DRY RUN) Started restore" unless opts[:quiet]
+    else
+      puts "Started restore" unless opts[:quiet]
+    end
+
     # validate backup path
     if !Dir.exists?(path)
       raise Error, "backup path '#{backup_dir}' does not exist!"
@@ -190,16 +205,9 @@ module ElasticUtil
 
     # ping it first
     uri = URI(url)
-    response = Net::HTTP.get_response(uri)
-    http = Net::HTTP.new(uri.host, uri.port)
-    http.read_timeout = 5
-    http.open_timeout = 5
-    response = http.start() {|http|
-      http.get("/")
-    }
-
+    response = api_get(uri)
     if !response.is_a?(Net::HTTPSuccess)
-      raise Error, "Unable to reach elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
+      raise Error, "Unable to reach Elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
     end
 
     # find files to import
@@ -210,19 +218,26 @@ module ElasticUtil
       puts "Found #{found_files.size} files to import" unless opts[:quiet]
     end
 
+    if opts[:dry]
+      found_files.each_with_index do |file, i|
+        puts "(#{i+1}/#{found_files.size}) bulk importing file #{file}" unless opts[:quiet]
+      end
+      puts "(DRY RUN) Finished restore of Elasticsearch #{url} with backup #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
+      return 0
+    end
+
     # bulk api request for each file
     found_files.each_with_index do |file, i|
       puts "(#{i+1}/#{found_files.size}) bulk importing file #{file}" unless opts[:quiet]
       payload = File.read(file)
-      # uri = URI(url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      response = http.post("/_bulk", payload)
+      uri = URI(url + "/_bulk")
+      response = api_post(uri, payload, {:headers => {"Content-Type" => "application/x-ndjson"} })
       if !response.is_a?(Net::HTTPSuccess)
         raise Error, "HTTP request failure!\n#{response.inspect}\n#{response.body.to_s}"
       end
     end
 
-    puts "Finished restore of elasticsearch #{url} with backup #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
+    puts "Finished restore of Elasticsearch #{url} with backup #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
     return true
   end
   
@@ -235,8 +250,12 @@ module ElasticUtil
         FileUtils.mkdir_p(dir_name)
         file_name = File.join(dir_name, index_type) + (file_index ? "_#{file_index}" : "") + ".json.data"
         # prepare record for bulk api injection
+        doc_type = hit['_type']
+        if opts[:replace_types] && opts[:replace_types][doc_type]
+          doc_type = opts[:replace_types][doc_type]
+        end
         action_json = {'index' => {
-          '_index' => hit['_index'], '_type' => hit['_type'], '_id' => hit['_id']
+          '_index' => hit['_index'], '_type' => doc_type, '_id' => hit['_id']
         } }
         source_json = hit['_source']
         if opts[:exclude_fields] && source_json
@@ -244,6 +263,7 @@ module ElasticUtil
             source_json.delete(field)
           end
         end
+
         File.open(file_name, 'a') do |file|
           file.write JSON.generate(action_json) + "\n" + JSON.generate(source_json) + "\n"
         end
@@ -251,4 +271,58 @@ module ElasticUtil
     end
   end
 
+  def self.exec_request(uri, http_method="GET", opts={})
+    # parse request URI and options
+    uri = uri.is_a?(URI) ? uri : URI(uri)
+    http_method = http_method.to_s.upcase
+    headers = opts[:headers] || {}
+    payload = opts[:payload] || opts[:body]
+    http = Net::HTTP.new(uri.host, uri.port)
+    if uri.scheme == 'https'
+      http.use_ssl = true
+      # todo: always ignore ssl errors for now, but this should be an option
+      # http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+    http.read_timeout = opts[:read_timeout] || (60*15)
+    http.open_timeout = opts[:open_timeout] || 10
+    request = nil
+    if http_method == "GET"
+      request = Net::HTTP::Get.new uri.request_uri
+    elsif http_method == "POST"
+      request = Net::HTTP::Post.new uri.request_uri
+      request.body = payload if payload
+    elsif http_method == "PUT"
+      request = Net::HTTP::Put.new uri.request_uri
+      request.body = payload if payload
+    elsif http_method == "DELETE"
+      request = Net::HTTP::Delete.new uri.request_uri
+    else
+      raise "HTTP method is unknown: '#{http_method}'"
+    end
+    # set headers
+    headers.each { |k,v| request[k] = v }
+    # todo: set default Accept: application/json  (probably, right?)
+    # set default Content-Type
+    if payload && headers['Content-Type'].nil?
+      headers['Content-Type'] = "application/json"
+    end
+    # set basic auth
+    if uri.user
+      request.basic_auth uri.user, uri.password
+    end
+    # execute request
+    response = http.request(request)
+    # return the resulting Net::HTTPResponse
+    return response
+  end
+
+  def self.api_get(uri, opts={})
+    exec_request(uri, "GET", opts={})
+  end
+
+  def self.api_post(uri, payload, opts={})
+    exec_request(uri, "POST", opts.merge({payload:payload}))
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastic-util
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.8
 platform: ruby
 authors:
 - James Dickson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-08 00:00:00.000000000 Z
+date: 2020-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,8 +38,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ElasticUtil uses ElasticSearch's scroll and _bulk APIs to dump and restore
-  indices
+description: ElasticUtil uses ElasticSearch's /_search and /_bulk APIs to dump and
+  restore indices
 email:
 - dickson.james@gmail.com
 executables:
@@ -49,6 +49,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - Gemfile
+- HISTORY.md
 - README.md
 - Rakefile
 - bin/elastic-util
@@ -75,7 +76,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Provides backup and restore for ElasticSearch data