es_dump_restore 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6484c47c7bc3f4b8d77671791b34471f0a2767d5
-  data.tar.gz: 70e2b1056a94d7a5168d930ce2d190ebaa4b618b
+  metadata.gz: 92d15eebfd8d46e234d93dc0868dfbfe0f69a2a0
+  data.tar.gz: 937b84068b58f036d19befa6c97df87aa4b390cb
 SHA512:
-  metadata.gz: 48279c3090efc690a1df0c1f3eaac017890b4e0dbe8992314aff007b57f2b1438dbdddbfe78d6c2332d57ec2e8ca89780c200f4183b6bcf9f8b1286ff9b4e78c
-  data.tar.gz: 1a06773b02e7b7677a3b6eb4417900c723861481b9620d04e79936dbbb8957ff625f55cf923af679d49ed1774b350cbdd4a6b73deaf31853da6c7ee96e13dd27
+  metadata.gz: 2aef2931dfc1b07cc8ccd5f522737d433db672657430347af66b8a8143b8d821371bb90a0743d365fb26137ca518bb1a918d30d9a4a3059e9e401c2f098f3277
+  data.tar.gz: ae8c5562157a305db99ce44229eafee4ee958b587ba4e9bd624d7306f932227805f8033473c61a1de5891857274b4086beb6bc86d7be85a44dc896a3d5d5ecfd

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# Version 2.2.0 - October 20, 2017
+
+* Add an optional command-line parameter for retries on timeout errors (thanks to David Henry!)
+
 # Version 2.1.0 - September 30, 2015
 
 * Dump index metadata along with data

data/README.md

@@ -23,11 +23,11 @@ To dump an ElasticSearch index by type to a file:
 
 To restore an index to an ElasticSearch server:
 
-    es_dump_restore restore ELASTIC_SEARCH_SERVER_URL DESTINATION_INDEX FILENAME_ZIP [SETTING_OVERRIDES] [BATCH_SIZE]
+    es_dump_restore restore ELASTIC_SEARCH_SERVER_URL DESTINATION_INDEX FILENAME_ZIP [SETTING_OVERRIDES] [BATCH_SIZE] [EXCEPTION_RETRIES]
 
 To restore an index and set an alias to point to it:
 
-    es_dump_restore restore_alias ELASTIC_SEARCH_SERVER_URL DESTINATION_ALIAS DESTINATION_INDEX FILENAME_ZIP [SETTING_OVERRIDES] [BATCH_SIZE]
+    es_dump_restore restore_alias ELASTIC_SEARCH_SERVER_URL DESTINATION_ALIAS DESTINATION_INDEX FILENAME_ZIP [SETTING_OVERRIDES] [BATCH_SIZE] [EXCEPTION_RETRIES]
 
 This loads the dump into an index named `DESTINATION_INDEX`, and once the load
 is complete sets the alias `DESTINATION_ALIAS` to point to it.  If
@@ -53,6 +53,9 @@ documents which will be sent to elasticsearch at once.  This defaults to 1000,
 which is normally fine, but if you have particularly complex documents or
 mappings this might need reducing to avoid timeouts.
 
+If `EXCEPTION_RETRIES` is set to an integer it tells the bulk load process how
+many times is should retry when a timeout is raised. This defaults to 1.
+
 ## Contributing
 
 1. Fork it

data/lib/es_dump_restore/app.rb

@@ -26,8 +26,8 @@ module EsDumpRestore
     end
 
     desc "restore URL INDEX_NAME FILENAME", "Restores a dumpfile into the given ElasticSearch index"
-    def restore(url, index_name, filename, overrides = nil, batch_size = 1000)
-      client = EsClient.new(url, index_name, nil)
+    def restore(url, index_name, filename, overrides = nil, batch_size = 1000, exception_retries = 1)
+      client = EsClient.new(url, index_name, nil, exception_retries)
 
       Dumpfile.read(filename) do |dumpfile|
         client.create_index(dumpfile.index, overrides)
@@ -42,8 +42,8 @@ module EsDumpRestore
 
     desc "restore_alias URL ALIAS_NAME INDEX_NAME FILENAME", "Restores a dumpfile into the given ElasticSearch index, and then sets the alias to point at that index, removing any existing indexes pointed at by the alias"
     def restore_alias(url, alias_name, index_name, filename, overrides = nil,
-                      batch_size = 1000)
-      client = EsClient.new(url, index_name, nil)
+                      batch_size = 1000, exception_retries = 1)
+      client = EsClient.new(url, index_name, nil, exception_retries)
       client.check_alias alias_name
 
       Dumpfile.read(filename) do |dumpfile|

data/lib/es_dump_restore/es_client.rb

@@ -7,12 +7,13 @@ module EsDumpRestore
     attr_accessor :base_uri
     attr_accessor :index_name
 
-    def initialize(base_uri, index_name, type)
+    def initialize(base_uri, index_name, type, exception_retries=1)
       @httpclient = HTTPClient.new
       @index_name = index_name
 
       @es_uri = base_uri
       @path_prefix = type.nil? ? index_name : index_name + "/" + type
+      @exception_retries = exception_retries
     end
 
     def mappings
@@ -124,7 +125,17 @@ module EsDumpRestore
     end
 
     def bulk_index(data)
-      request(:post, "#{@path_prefix}/_bulk", :body => data)
+      retries = 0
+      begin
+        request(:post, "#{@path_prefix}/_bulk", :body => data)
+      rescue HTTPClient::TimeoutError => e
+        if retries < @exception_retries
+          retries += 1
+          puts "Retrying (#{retries} of #{@exception_retries}) '#{@path_prefix}/_bulk'"
+          retry
+        end
+        raise e
+      end
     end
 
     private

data/lib/es_dump_restore/version.rb

@@ -1,3 +1,3 @@
 module EsDumpRestore
-  VERSION = "2.1.0"
+  VERSION = "2.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: es_dump_restore
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Nat Budin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-30 00:00:00.000000000 Z
+date: 2017-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -135,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: Dump ElasticSearch indexes to files and restore them back