es_dump_restore 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 26b95784a5cbe09598686a716498b43feb08e905
-  data.tar.gz: fc3451f6a37dda8116891bb9daa09291faaaf485
+  metadata.gz: a45b3f41110e68aa160d81d3094501b4db2d914a
+  data.tar.gz: 127abfad968e31cfd7fb1ea1d933dd55d9497839
 SHA512:
-  metadata.gz: dd7c79f8d006bd3ab5ac8818ab5a86b8a9d81bd9e2819b17d7d42a45cd8347d96d15b6e2bcb3d5ff880db9431c87d25e9223d9e654b1dadee55029bfd7f18db4
-  data.tar.gz: db90aeae9f731d02009f9cd857708cd6eac25ac135db5558adcf47cdea4bb3e73d37b89a39b44a463f7fa3fc113519854470719c3f36606e1385615284595361
+  metadata.gz: 7a888f720b201ec78d4a6b1c2ca75033f2fd98e07838282a3180fe3a3488d8d39faa4971073f4e5b027049e0f01ec038ef0a274cf710a89f07856241e4c564e4
+  data.tar.gz: ca426d7becd4d7f59d2708d926a38aae42a56a23a4bd9b2b7657f65c5f860897e4da5578175cbe971b04fd48d3898b5de19ff7d2a1fe3c40ff5e8fcc5f2d88cb

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# Version 1.2.0 - June 24, 2015
+
+* Allow overriding the batch size on the command line (thanks to Richard Boulton once again!)
+
 # Version 1.1.0 - May 12, 2015
 
 * Allow specifying additional index settings to the `restore` and `restore_alias` commands (thanks again Richard Boulton!)

data/README.md

@@ -23,11 +23,11 @@ To dump an ElasticSearch index by type to a file:
 
 To restore an index to an ElasticSearch server:
 
-    es_dump_restore restore ELASTIC_SEARCH_SERVER_URL DESTINATION_INDEX FILENAME_ZIP [SETTING_OVERRIDES]
+    es_dump_restore restore ELASTIC_SEARCH_SERVER_URL DESTINATION_INDEX FILENAME_ZIP [SETTING_OVERRIDES] [BATCH_SIZE]
 
 To restore an index and set an alias to point to it:
 
-    es_dump_restore restore_alias ELASTIC_SEARCH_SERVER_URL DESTINATION_ALIAS DESTINATION_INDEX FILENAME_ZIP [SETTING_OVERRIDES]
+    es_dump_restore restore_alias ELASTIC_SEARCH_SERVER_URL DESTINATION_ALIAS DESTINATION_INDEX FILENAME_ZIP [SETTING_OVERRIDES] [BATCH_SIZE]
 
 This loads the dump into an index named `DESTINATION_INDEX`, and once the load
 is complete sets the alias `DESTINATION_ALIAS` to point to it.  If
@@ -48,6 +48,11 @@ would read the dump file `test_dump.zip`, load it into an index called
 would be set to have no replicas, and only 1 shard, but have all other settings
 from the dump file.
 
+If `BATCH_SIZE` is set for a restore command, it controls the number of
+documents which will be sent to elasticsearch at once.  This defaults to 1000,
+which is normally fine, but if you have particularly complex documents or
+mappings this might need reducing to avoid timeouts.
+
 ## Contributing
 
 1. Fork it

data/lib/es_dump_restore/app.rb

@@ -61,14 +61,14 @@ module EsDumpRestore
     end
 
     desc "restore URL INDEX_NAME FILENAME", "Restores a dumpfile into the given ElasticSearch index"
-    def restore(url, index_name, filename, overrides=nil)
+    def restore(url, index_name, filename, overrides = nil, batch_size = 1000)
       client = EsClient.new(url, index_name, nil)
 
       Dumpfile.read(filename) do |dumpfile|
         client.create_index(dumpfile.index, overrides)
 
         bar = ProgressBar.new(dumpfile.num_objects) unless options[:noprogressbar]
-        dumpfile.scan_objects(1000) do |batch, size|
+        dumpfile.scan_objects(batch_size.to_i) do |batch, size|
           client.bulk_index batch
           bar.increment!(size) unless options[:noprogressbar]
         end
@@ -76,7 +76,8 @@ module EsDumpRestore
     end
 
     desc "restore_alias URL ALIAS_NAME INDEX_NAME FILENAME", "Restores a dumpfile into the given ElasticSearch index, and then sets the alias to point at that index, removing any existing indexes pointed at by the alias"
-    def restore_alias(url, alias_name, index_name, filename, overrides=nil)
+    def restore_alias(url, alias_name, index_name, filename, overrides = nil,
+                      batch_size = 1000)
       client = EsClient.new(url, index_name, nil)
       client.check_alias alias_name
 
@@ -84,7 +85,7 @@ module EsDumpRestore
         client.create_index(dumpfile.index, overrides)
 
         bar = ProgressBar.new(dumpfile.num_objects) unless options[:noprogressbar]
-        dumpfile.scan_objects(1000) do |batch, size|
+        dumpfile.scan_objects(batch_size.to_i) do |batch, size|
           client.bulk_index batch
           bar.increment!(size) unless options[:noprogressbar]
         end

data/lib/es_dump_restore/version.rb

@@ -1,3 +1,3 @@
 module EsDumpRestore
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: es_dump_restore
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Nat Budin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-12 00:00:00.000000000 Z
+date: 2015-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json