es_dump_restore 0.0.13 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 86d387c4543740231133cf902eb7ef3ccc4b8032
-  data.tar.gz: f713de78847fab41c950625668dbfbfb3bc64241
+  metadata.gz: 8c543de6aac0bbd4e8380b4fdf8b9f995c2ff33c
+  data.tar.gz: eb25e1154138dba1bb53c71080190c9a3d57a7c2
 SHA512:
-  metadata.gz: 0b4c08c8beac9653c832bd34b4fa544934f932eb1f7a06c124d6a824a9287e85f3efbe07b386e1880c2cff1ce47da57eecacf6f7595bf5789bacdf7ab9cc7fb5
-  data.tar.gz: 92889a08a395f142596394637b886fac7cd1baa7b23f09873ee69f70007f69fcaee5ba5743888a7d96d3bc2b9da5648fd45c538927ba88091de7a5069e309e0a
+  metadata.gz: 5af4422476d43ff85ea997463162703f4c4d214cd617517ea4ca9950c9ef24c06367b7fa8eeeadc0acc683f0ee1ab352d250c969ae8eef3263a5b9ce9e94533c
+  data.tar.gz: 6707141cbeae98732da6dd62d786f6c2ba2c09879f29a5b9c027559f73dca4ac588d50b7c68af71060f541a4ae7e62683633f92ddfa61a29fcf625cd340f458d

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# Version 1.0.0 - April 11, 2015
+
+* Add a `restore_alias` command, which performs a restore and switches an alias to point at the result (thanks Richard Boulton!)
+* Fixes a bug preventing index restoration on newer ES releases (thanks again Richard Boulton!)
+* Richard Boulton tells us he's used `es_dump_restore` to perform a migration on the www.gov.uk indexes, so we think it's high time we started calling this tool ready for prime time.  Announcing 1.0.0!
+
 # Version 0.0.13 - April 10, 2015
 
 * Fix a regression introduced for ES version 0.90.x by the previous release (thanks again Richard Boulton!)

data/lib/es_dump_restore/app.rb

@@ -75,5 +75,23 @@ module EsDumpRestore
       end
     end
 
+    desc "restore_alias URL ALIAS_NAME INDEX_NAME FILENAME", "Restores a dumpfile into the given ElasticSearch index, and then sets the alias to point at that index, removing any existing indexes pointed at by the alias"
+    def restore_alias(url, alias_name, index_name, filename)
+      client = EsClient.new(url, index_name, nil)
+      client.check_alias alias_name
+
+      Dumpfile.read(filename) do |dumpfile|
+        client.create_index(dumpfile.index)
+
+        bar = ProgressBar.new(dumpfile.num_objects) unless options[:noprogressbar]
+        dumpfile.scan_objects(1000) do |batch, size|
+          client.bulk_index batch
+          bar.increment!(size) unless options[:noprogressbar]
+        end
+      end
+
+      client.replace_alias_and_close_old_index alias_name
+    end
+
   end
 end

data/lib/es_dump_restore/es_client.rb

@@ -74,11 +74,50 @@ module EsDumpRestore
     end
 
     def create_index(metadata)
-      request(:post, "", :body => MultiJson.dump(metadata))
+      request(:post, "#{@path_prefix}", :body => MultiJson.dump(metadata))
+    end
+
+    def check_alias(alias_name)
+      # Checks that it's possible to do an atomic restore using the given alias
+      # name.  This requires that:
+      #  - `alias_name` doesn't point to an existing index
+      #  - `index_name` doesn't point to an existing index
+      existing = request(:get, "_aliases")
+      if existing.include? index_name
+        raise "There is already an index called #{index_name}"
+      end
+      if existing.include? alias_name
+        raise "There is already an index called #{alias_name}"
+      end
+    end
+
+    def replace_alias_and_close_old_index(alias_name)
+      existing = request(:get, "_aliases")
+
+      # Response of the form:
+      #   { "index_name" => { "aliases" => { "a1" => {}, "a2" => {} } } }
+      old_aliased_indices = existing.select { |name, details|
+        details.fetch("aliases", {}).keys.include? alias_name
+      }
+      old_aliased_indices = old_aliased_indices.keys
+
+      # For any existing indices with this alias, remove the alias
+      # We would normally expect 0 or 1 such index, but several is
+      # valid too
+      actions = old_aliased_indices.map { |old_index_name|
+        { "remove" => { "index" => old_index_name, "alias" => alias_name } }
+      }
+
+      actions << { "add" => { "index" => index_name, "alias" => alias_name } }
+
+      request(:post, "_aliases", :body => MultiJson.dump({ "actions" => actions }))
+      old_aliased_indices.each do |old_index_name|
+        request(:post, "#{old_index_name}/_close")
+      end
     end
 
     def bulk_index(data)
-      request(:post, "_bulk", :body => data)
+      request(:post, "#{@path_prefix}/_bulk", :body => data)
     end
 
     private
@@ -88,12 +127,11 @@ module EsDumpRestore
       begin
         response = @httpclient.request(method, request_uri, options)
         unless response.ok? or extra_allowed_exitcodes.include? response.status
-          raise "Request failed with status #{response.status}: #{response.reason}"
+          raise "Request failed with status #{response.status}: #{response.reason} #{response.content}"
         end
         MultiJson.load(response.content)
       rescue Exception => e
         puts "Exception caught issuing HTTP request to #{request_uri}"
-        puts "options: #{options}"
         raise e
       end
     end

data/lib/es_dump_restore/version.rb

@@ -1,3 +1,3 @@
 module EsDumpRestore
-  VERSION = "0.0.13"
+  VERSION = "1.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: es_dump_restore
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 1.0.0
 platform: ruby
 authors:
 - Nat Budin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-10 00:00:00.000000000 Z
+date: 2015-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json