elastic-util 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af7028b9a21af54dc8f66849ce7a22e9739dbf0a2b7d8d2fd4091144337b3fda
-  data.tar.gz: 8caf084b468de28607516b1c8d8643de9373732a5d1628db4618307f6b7dac64
+  metadata.gz: b30dc1de09489fede6a09227ff9a8a292f416fdcf8e9eb041bcb46c08cbd6767
+  data.tar.gz: 5b604aafcae78e703530acec175bf1235178cb784711c2bcb8a665b9849c7792
 SHA512:
-  metadata.gz: 07ce1c430cd062388456fc8d19717546457aba8bd20952a54697d17192a36e63367d66a37f78f8fc08d48d0feb89fd5c26c53fccc32a0f5df26e65e31278bc73
-  data.tar.gz: bb50a5ada50644ee7edaf831af2d1966d4610515d268310084a01c0d0f278c2679c3a6757fc62ee6c6863c5f53c03842a2190335ed807037c97d7cd2eb004a38
+  metadata.gz: 4d4f94be2ca1d450ea260ab4d6f8450e8dfb8025f52f72281a932dc33480fdc0a965175166d9aa50000d386a59c8b65347b02aac6472e68432f663798f2d2e0b
+  data.tar.gz: '0029396b7a2bc8960f10532a8bdec303e4e3e813c8acbbab795c4dffbaf0e36cb707cf311acd363a5c34237b3ce7ea0454a37c1d261d491cc6b6919a64c6da8e'

data/HISTORY.md

@@ -0,0 +1,14 @@
+# History
+
+This is a changelog for the changes made to the `elastic-util` gem.
+
+### v0.1.8
+* improved `--help` output and error messages
+
+### v0.1.7
+* basic auth support inside the url argument using the format username:password@url
+* https support
+
+### v0.1.6
+* New backup option --replace-types
+

data/bin/elastic-util

@@ -3,17 +3,25 @@ require 'elastic-util'
 require 'optparse'
 
 prog_name = "elastic-util" # $0.split('/').last
-usage = "Usage: #{prog_name} [backup|restore] [options]"
+prog_usage = "usage: #{prog_name} [backup|restore] [url] [directory] [options]"
 args = ARGV.dup
 command_name = args.shift
 
 case command_name
 
 when "backup"
-
+  banner = <<-EOT
+usage: #{prog_name} #{command_name} [url] [directory] [options]
+EOT
+  footer = <<-EOT
+The backup command provides way to dump Elasticsearch index data.
+The /_search API is used to paginate requests for each index being exported.
+The data is written to specified directory, with a file for each api request made.
+Example: #{prog_name} #{command_name} http://localhost:9300 /tmp/local-elastic-data
+EOT
   options = {}
   optparse = OptionParser.new do |opts|
-    opts.banner = "Usage: #{prog_name} #{command_name} [url] [directory] [options]"
+    opts.banner = banner
     opts.on('--indices x,y,z', Array, "The indices to backup. Default is all.") do |val|
       options[:indices] = val.collect {|it| it.strip }
     end
@@ -44,27 +52,24 @@ when "backup"
     opts.on( '-q', '--quiet', "Don't print to stdout. Default is false." ) do |val|
       options[:quiet] = true
     end
+    opts.on( '-d', '--dry-run', "Don't actually backup data, just print what would have happened. Default is false." ) do |val|
+      options[:dry] = true
+    end
     opts.on('-h', '--help', "Prints this help" ) do
-      puts opts
-      exit
+      puts opts, footer
+      exit 0
     end
   end
   optparse.parse!(args)
-  url = args[0]
-  backup_directory = args[1]
-  if url.nil? || url.empty?
-    $stderr.puts "#{prog_name}: missing required argument [url]"
-    $stderr.puts optparse
-    exit 1
-  end
-  if backup_directory.nil? || backup_directory.empty?
-    $stderr.puts "#{prog_name}: missing required argument [directory]"
-    $stderr.puts optparse
+  if args.count != 2
+    $stderr.puts "#{prog_name}: wrong number of arguments. Expected 2 and got #{args.count}: #{args.join(', ')}"
+    # $stderr.puts optparse
+    $stderr.puts banner
+    $stderr.puts "See `#{prog_name} #{command_name} --help` for more usage information."
     exit 1
   end
-  
   begin
-    result = ElasticUtil.backup(url, backup_directory, options)
+    result = ElasticUtil.backup(args[0], args[1], options)
     exit 0
   rescue ElasticUtil::Error => err
     $stderr.puts "#{prog_name}: #{err.message}"
@@ -74,34 +79,39 @@ when "backup"
 
 
 when "restore"
-
+  banner = <<-EOT
+usage: #{prog_name} #{command_name} [url] [directory] [options]
+EOT
+  footer = <<-EOT
+The restore command provides way to restore Elasticsearch index data.
+The /_bulk API is used to upload index data from the specified directory.
+The directory should contain data generated by the `#{prog_name} backup` command.
+e.g. #{prog_name} #{command_name} http://localhost:9400 /tmp/local-elastic-data
+EOT
   options = {}
   optparse = OptionParser.new do |opts|
-    opts.banner = "Usage: #{prog_name} #{command_name} [url] [directory] [options]"
+    opts.banner = banner
     opts.on( '-q', '--quiet', "Don't print to stdout. Default is false." ) do |val|
       options[:quiet] = true
     end
+    opts.on( '-d', '--dry-run', "Don't actually restore data, just print what would have happened. Default is false." ) do |val|
+      options[:dry] = true
+    end
     opts.on('-h', '--help', "Prints this help" ) do
-      puts opts
-      exit
+      puts opts, footer
+      exit 0
     end
   end
   optparse.parse!(args)
-  url = args[0]
-  backup_directory = args[1]
-  if url.nil? || url.empty?
-    $stderr.puts "#{prog_name}: missing required argument [url]"
-    $stderr.puts optparse
-    exit 1
-  end
-  if backup_directory.nil? || backup_directory.empty?
-    $stderr.puts "#{prog_name}: missing required argument [directory]"
-    $stderr.puts optparse
+  if args.count != 2
+    $stderr.puts "#{prog_name}: wrong number of arguments. Expected 2 and got #{args.count}: #{args.join(', ')}"
+    # $stderr.puts optparse
+    $stderr.puts banner
+    $stderr.puts "See `#{prog_name} #{command_name} --help` for more usage information."
     exit 1
   end
-  
   begin
-    result = ElasticUtil.restore(url, backup_directory, options)
+    result = ElasticUtil.restore(args[0], args[1], options)
     exit 0
   rescue ElasticUtil::Error => err
     $stderr.puts "#{prog_name}: #{err.message}"
@@ -109,10 +119,28 @@ when "restore"
     exit 1
   end
 
-when "-v"
+when "-h", "--help"
+  puts prog_usage
+  puts <<-EOT
+The elastic-util command provides way to dump and restore Elasticsearch index data.
+Example: #{prog_name} backup  http://localhost:9300 /tmp/local-elastic-data
+         #{prog_name} restore http://localhost:9400 /tmp/local-elastic-data
+EOT
+  exit 0
+when "-v","--version","version"
   puts ElasticUtil::VERSION
-else
-  $stderr.puts usage
+when "", nil
+  $stderr.puts "#{prog_name}: missing required argument [backup|restore]"
+  # $stderr.puts "Commands:"
+  # $stderr.puts "\tbackup"
+  # $stderr.puts "\trestore"
+  $stderr.puts prog_usage
+  $stderr.puts "See `#{prog_name} #{command_name} --help` for more usage information."
   exit 1
+else
+  # $stderr.puts "#{prog_name}: '#{command_name}' is not a recognized elastic-util command. See `#{prog_name} --help`"
+  $stderr.puts "#{prog_name}: '#{command_name}' is not a recognized elastic-util command."
+  $stderr.puts prog_usage
+  exit 3
 end
 

data/elastic-util.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["James Dickson"]
   spec.email         = ["dickson.james@gmail.com"]
   spec.summary       = "Provides backup and restore for ElasticSearch data"
-  spec.description   = "ElasticUtil uses ElasticSearch's scroll and _bulk APIs to dump and restore indices"
+  spec.description   = "ElasticUtil uses ElasticSearch's /_search and /_bulk APIs to dump and restore indices"
   #spec.homepage      = "http://www.elastic-util.com"
   spec.license       = "MIT"
 

data/lib/elastic_util.rb

@@ -4,16 +4,16 @@ require 'openssl'
 require 'json'
 require 'fileutils'
 
-# This module provides a way to backup and restore elasticsearch data.
+# This module provides a way to backup and restore Elasticsearch data.
 #
 # @example Backup data from one elasticsearch cluster and restore it to another.
 #
-#     ElasticUtil.backup('http://localhost:9200', '/tmp/mybackup', {size:5000})
-#     ElasticUtil.restore('http://localhost:9201', '/tmp/mybackup')
+#     ElasticUtil.backup('http://localhost:9300', '/tmp/local-elastic-data', {size:5000})
+#     ElasticUtil.restore('http://localhost:9301', '/tmp/local-elastic-data')
 #
 module ElasticUtil
 
-  VERSION = "0.1.7"
+  VERSION = "0.1.8"
 
   # The name of the data directory, relative to the user provided backup directory.
   DUMP_DIR = "es_data"
@@ -21,24 +21,23 @@ module ElasticUtil
   # A class to be raised for any known error condition.
   class Error < StandardError; end
 
-  # Backup elasticsearch data to a local directory.
+  # Backup Elasticsearch data to a local directory.
   #
   # This uses ElasticSearch's scroll api to fetch all records for indices
   # and write the data to a local directory. The files it generates are given a 
   # .json.data extension. They are not valid JSON files, but rather are in the 
   # format expected by ElasticSearch's _bulk api.
   #
-  # So #restore simply has to POST the contents of each file.
+  # So #restore simply has to POST the contents of each file as Content-Type: application/x-ndjson
   #
-  # Use the :size option to change the number or results to fetch at once,
-  # and also the size of the data files generated.
-  # The latter correlates to the of the the api requests made in #restore.
+  # Use the :size option to change the number of results to fetch at once and also the size of the data files generated.
+  # Increasing size means larger files and fewer api requests for both #backup and the subsequent #restore of that data.
   #
-  # @example Backup default elasticsearch running locally.
+  # @example Backup default Elasticsearch running locally.
   #
-  #     ElasticUtil.backup('http://localhost:9200', '/tmp/mybackup')
+  #     ElasticUtil.backup('http://localhost:9300', '/tmp/local-elastic-data')
   #
-  # @param [String] url The url of the elasticsearch cluster eg. 'http://localhost:9200'
+  # @param [String] url The url of the Elasticsearch cluster eg. 'http://localhost:9300'
   # @param [String] backup_dir The local directory to store data in. eg. '/tmp/es2.4'
   # @param [Hash] opts The options for this backup.
   # @option opts [Array] :indices The indices to backup. Default is all.
@@ -58,12 +57,19 @@ module ElasticUtil
     backup_dir = backup_dir.strip
     path = File.join(backup_dir.strip, DUMP_DIR)
     indices = []
+
+    if opts[:dry]
+      puts "(DRY RUN) Started backup" unless opts[:quiet]
+    else
+      puts "Started backup" unless opts[:quiet]
+    end
+
     # ping it first
     response = nil
     uri = URI(url)
     response = api_get(uri)
     if !response.is_a?(Net::HTTPSuccess)
-      raise Error, "Unable to reach elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
+      raise Error, "Unable to reach Elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
     end
 
     # determine indices to backup, default is everything.
@@ -107,6 +113,14 @@ module ElasticUtil
     end
     FileUtils.mkdir_p(path)
 
+    if opts[:dry]
+      indices.each_with_index do |index_name, i|
+        puts "(#{i+1}/#{indices.size}) backing up index #{index_name}" unless opts[:quiet]
+      end
+      puts "(DRY RUN) Finished backup of Elasticsearch #{url} to directory #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
+      return 0
+    end
+
     # dump data
     indices.each_with_index do |index_name, i|
       puts "(#{i+1}/#{indices.size}) backing up index #{index_name}" unless opts[:quiet]
@@ -154,18 +168,18 @@ module ElasticUtil
       end
     end
     
-    puts "Finished backup of elasticsearch #{url} to directory #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
+    puts "Finished backup of Elasticsearch #{url} to directory #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
     return true
   end
 
-  # Restore elasticsearch data from a backup.
+  # Restore Elasticsearch data from a backup.
   # This will do a POST to the _bulk api for each file in the backup directory.
   #
   # @example Restore local cluster with our backup.
   #
-  #     ElasticUtil.restore('http://localhost:9201', '/tmp/mybackup')
+  #     ElasticUtil.restore('http://localhost:9301', '/tmp/local-elastic-data')
   #
-  # @param [String] url The url of the elasticsearch cluster eg. 'http://localhost:9200'.
+  # @param [String] url The url of the Elasticsearch cluster eg. 'http://localhost:9200'.
   # @param [String] backup_dir The backup directory.
   # @param [Hash] opts The options for this backup.
   # @option opts [true] :quiet Don't print anything. Default is false.
@@ -174,9 +188,16 @@ module ElasticUtil
   #
   def self.restore(url, backup_dir, opts={})
     start_time = Time.now
+    url = url.strip.chomp("/")
     backup_dir = backup_dir.strip
     path = File.join(backup_dir.strip, DUMP_DIR)
 
+    if opts[:dry]
+      puts "(DRY RUN) Started restore" unless opts[:quiet]
+    else
+      puts "Started restore" unless opts[:quiet]
+    end
+
     # validate backup path
     if !Dir.exists?(path)
       raise Error, "backup path '#{backup_dir}' does not exist!"
@@ -186,7 +207,7 @@ module ElasticUtil
     uri = URI(url)
     response = api_get(uri)
     if !response.is_a?(Net::HTTPSuccess)
-      raise Error, "Unable to reach elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
+      raise Error, "Unable to reach Elasticsearch at url '#{url}'!\n#{response.inspect}\n#{response.body.to_s}"
     end
 
     # find files to import
@@ -197,6 +218,14 @@ module ElasticUtil
       puts "Found #{found_files.size} files to import" unless opts[:quiet]
     end
 
+    if opts[:dry]
+      found_files.each_with_index do |file, i|
+        puts "(#{i+1}/#{found_files.size}) bulk importing file #{file}" unless opts[:quiet]
+      end
+      puts "(DRY RUN) Finished restore of Elasticsearch #{url} with backup #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
+      return 0
+    end
+
     # bulk api request for each file
     found_files.each_with_index do |file, i|
       puts "(#{i+1}/#{found_files.size}) bulk importing file #{file}" unless opts[:quiet]
@@ -208,7 +237,7 @@ module ElasticUtil
       end
     end
 
-    puts "Finished restore of elasticsearch #{url} with backup #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
+    puts "Finished restore of Elasticsearch #{url} with backup #{backup_dir} (took #{(Time.now-start_time).round(3)}s)" unless opts[:quiet]
     return true
   end
   
@@ -242,7 +271,12 @@ module ElasticUtil
     end
   end
 
-  def self.api_get(uri, opts={})
+  def self.exec_request(uri, http_method="GET", opts={})
+    # parse request URI and options
+    uri = uri.is_a?(URI) ? uri : URI(uri)
+    http_method = http_method.to_s.upcase
+    headers = opts[:headers] || {}
+    payload = opts[:payload] || opts[:body]
     http = Net::HTTP.new(uri.host, uri.port)
     if uri.scheme == 'https'
       http.use_ssl = true
@@ -251,42 +285,44 @@ module ElasticUtil
       http.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
     http.read_timeout = opts[:read_timeout] || (60*15)
-    http.open_timeout = opts[:open_timeout] || 5
-    request = Net::HTTP::Get.new uri.request_uri
-    if opts[:headers]
-      opts[:headers].each do |k,v|
-        request[k] = v
-      end
+    http.open_timeout = opts[:open_timeout] || 10
+    request = nil
+    if http_method == "GET"
+      request = Net::HTTP::Get.new uri.request_uri
+    elsif http_method == "POST"
+      request = Net::HTTP::Post.new uri.request_uri
+      request.body = payload if payload
+    elsif http_method == "PUT"
+      request = Net::HTTP::Put.new uri.request_uri
+      request.body = payload if payload
+    elsif http_method == "DELETE"
+      request = Net::HTTP::Delete.new uri.request_uri
+    else
+      raise "HTTP method is unknown: '#{http_method}'"
+    end
+    # set headers
+    headers.each { |k,v| request[k] = v }
+    # todo: set default Accept: application/json  (probably, right?)
+    # set default Content-Type
+    if payload && headers['Content-Type'].nil?
+      headers['Content-Type'] = "application/json"
     end
+    # set basic auth
     if uri.user
       request.basic_auth uri.user, uri.password
     end
+    # execute request
     response = http.request(request)
+    # return the resulting Net::HTTPResponse
     return response
   end
 
+  def self.api_get(uri, opts={})
+    exec_request(uri, "GET", opts={})
+  end
+
   def self.api_post(uri, payload, opts={})
-    http = Net::HTTP.new(uri.host, uri.port)
-    if uri.scheme == 'https'
-      http.use_ssl = true
-      # todo: always ignore ssl errors for now, but this should be an option
-      # http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-    end
-    http.read_timeout = opts[:read_timeout] || (60*15)
-    http.open_timeout = opts[:open_timeout] || 5
-    request = Net::HTTP::Post.new uri.request_uri
-    if opts[:headers]
-      opts[:headers].each do |k,v|
-        request[k] = v
-      end
-    end
-    if uri.user
-      request.basic_auth uri.user, uri.password
-    end
-    request.body = payload
-    response = http.request(request)
-    return response
+    exec_request(uri, "POST", opts.merge({payload:payload}))
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastic-util
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - James Dickson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-19 00:00:00.000000000 Z
+date: 2020-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,8 +38,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ElasticUtil uses ElasticSearch's scroll and _bulk APIs to dump and restore
-  indices
+description: ElasticUtil uses ElasticSearch's /_search and /_bulk APIs to dump and
+  restore indices
 email:
 - dickson.james@gmail.com
 executables:
@@ -49,6 +49,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - Gemfile
+- HISTORY.md
 - README.md
 - Rakefile
 - bin/elastic-util