es_dump_restore 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4fd3daa61667baf72f9f0c62fe8b6f71e0e5c5de
-  data.tar.gz: a9d1eb03b685529be48895e92a856443e8cec280
+  metadata.gz: 6484c47c7bc3f4b8d77671791b34471f0a2767d5
+  data.tar.gz: 70e2b1056a94d7a5168d930ce2d190ebaa4b618b
 SHA512:
-  metadata.gz: 2a5728d62cfd999870e6d19b3ba4ad8350dce18d8ec860616c39d475cd01d39f27bf85fa429e4da75f6c2fb5d7c6d339c6a89289cc0a808f2aeaf67bc4d6e5d3
-  data.tar.gz: 6076595aba9cb8c3beec6259fd70fef81917ea8841115c7fb1c76e2e858c531b25f06d92fb0d785cf582c89095629e0387adaae34a63c6e5ef3591bd26b82c28
+  metadata.gz: 48279c3090efc690a1df0c1f3eaac017890b4e0dbe8992314aff007b57f2b1438dbdddbfe78d6c2332d57ec2e8ca89780c200f4183b6bcf9f8b1286ff9b4e78c
+  data.tar.gz: 1a06773b02e7b7677a3b6eb4417900c723861481b9620d04e79936dbbb8957ff625f55cf923af679d49ed1774b350cbdd4a6b73deaf31853da6c7ee96e13dd27

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# Version 2.1.0 - September 30, 2015
+
+* Dump index metadata along with data
+* Bugfix: correctly retrieve \_settings when using the dump_type action
+
 # Version 2.0.0 - July 1, 2015
 
 Thanks to Alex Tomlins for this feature contribution

data/lib/es_dump_restore/app.rb

@@ -16,51 +16,13 @@ module EsDumpRestore
     desc "dump URL INDEX_NAME FILENAME", "Creates a dumpfile based on the given ElasticSearch index"
     def dump(url, index_name, filename)
       client = EsClient.new(url, index_name, nil)
-
-      Dumpfile.write(filename) do |dumpfile|
-        dumpfile.index = {
-          settings: client.settings,
-          mappings: client.mappings
-        }
-
-        client.start_scan do |scroll_id, total|
-          dumpfile.num_objects = total
-          bar = ProgressBar.new(total) if options[:progressbar]
-
-          dumpfile.get_objects_output_stream do |out|
-            client.each_scroll_hit(scroll_id) do |hit|
-              metadata = { index: { _type: hit["_type"], _id: hit["_id"] } }
-              out.write("#{MultiJson.dump(metadata)}\n#{MultiJson.dump(hit["_source"])}\n")
-              bar.increment! if options[:progressbar]
-            end
-          end
-        end
-      end
+      _dump(client, filename)
     end
 
     desc "dump_type URL INDEX_NAME TYPE FILENAME", "Creates a dumpfile based on the given ElasticSearch index"
     def dump_type(url, index_name, type, filename)
       client = EsClient.new(url, index_name, type)
-
-      Dumpfile.write(filename) do |dumpfile|
-        dumpfile.index = {
-          settings: client.settings,
-          mappings: client.mappings
-        }
-
-        client.start_scan do |scroll_id, total|
-          dumpfile.num_objects = total
-          bar = ProgressBar.new(total) if options[:progressbar]
-
-          dumpfile.get_objects_output_stream do |out|
-            client.each_scroll_hit(scroll_id) do |hit|
-              metadata = { index: { _type: hit["_type"], _id: hit["_id"] } }
-              out.write("#{MultiJson.dump(metadata)}\n#{MultiJson.dump(hit["_source"])}\n")
-              bar.increment! if options[:progressbar]
-            end
-          end
-        end
-      end
+      _dump(client, filename)
     end
 
     desc "restore URL INDEX_NAME FILENAME", "Restores a dumpfile into the given ElasticSearch index"
@@ -97,5 +59,35 @@ module EsDumpRestore
       client.replace_alias_and_close_old_index alias_name
     end
 
+    private
+
+    def _dump(client, filename)
+      Dumpfile.write(filename) do |dumpfile|
+        dumpfile.index = {
+          settings: client.settings,
+          mappings: client.mappings
+        }
+
+        client.start_scan do |scroll_id, total|
+          dumpfile.num_objects = total
+          bar = ProgressBar.new(total) if options[:progressbar]
+
+          dumpfile.get_objects_output_stream do |out|
+            client.each_scroll_hit(scroll_id) do |hit|
+              hit['fields'] ||= {}
+              metadata = { index: { _type: hit["_type"], _id: hit["_id"] } }
+
+              %w(_timestamp _version _routing _percolate _parent _ttl).each do |metadata_field|
+                metadata[:index][metadata_field] = hit['fields'][metadata_field] if hit['fields'][metadata_field]
+              end
+
+              out.write("#{MultiJson.dump(metadata)}\n#{MultiJson.dump(hit["_source"])}\n")
+
+              bar.increment! if options[:progressbar]
+            end
+          end
+        end
+      end
+    end
   end
 end

data/lib/es_dump_restore/es_client.rb

@@ -29,7 +29,7 @@ module EsDumpRestore
     end
 
     def settings
-      data = request(:get, "#{@path_prefix}/_settings")
+      data = request(:get, "#{@index_name}/_settings")
       if data.values.size != 1
         raise "Unexpected response: #{data}"
       end
@@ -44,7 +44,10 @@ module EsDumpRestore
     def start_scan(&block)
       scroll = request(:get, "#{@path_prefix}/_search",
         query: { search_type: 'scan', scroll: '10m', size: 500 },
-        body: MultiJson.dump({ query: { match_all: {} } }) )
+        body: MultiJson.dump({
+          fields: ['_source', '_timestamp', '_version', '_routing', '_percolate', '_parent', '_ttl'],
+          query: { match_all: {} } }
+        ))
       total = scroll["hits"]["total"]
       scroll_id = scroll["_scroll_id"]
 

data/lib/es_dump_restore/version.rb

@@ -1,3 +1,3 @@
 module EsDumpRestore
-  VERSION = "2.0.0"
+  VERSION = "2.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: es_dump_restore
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Nat Budin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-01 00:00:00.000000000 Z
+date: 2015-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json