elasticsearch-extensions 0.0.22 → 0.0.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 33490d5d862da6d64e26435f1a77291e2a0a82d5
-  data.tar.gz: 7b57c8185dde5a4278c142c1de61605e3107ff39
+  metadata.gz: 929b42632d0c89e1007c16e6d08d465df863ad58
+  data.tar.gz: 8ebc114b63d1cf9d64346a7da06d4fc385a401f6
 SHA512:
-  metadata.gz: 36cee1857c1fbea9d28821cb54d425df5a38f28c9d23a2a3ebd827ad29cb9f7ec004405e8b0e85176a199355ddb0ae9650980988b036f209581197f9af71d58a
-  data.tar.gz: 4a8a001059bd6982c26a21663fceb8298711cc0b73ddefd17e7be26aa97cf0e540ca7f2a5327a274f1d3c700662ecd89c58eaa8ed954eda62975ec6f97b0771c
+  metadata.gz: 053e210e305b080280e75ef955a5a6b7981cadc604cab50250308ce6a335ea0ef06e5eef655e11789b78a501604bc7399a4b747e4db6ab292812c209a7cf014e
+  data.tar.gz: c19de28219676b133fe3850cadadd42087c9ebc3da86da055563e3e2be68c5bfa1e4d7c4aff79b68cdc629acc7dbf8674b751d4f97336558f8dff36f81deecdd

data/Gemfile

@@ -3,18 +3,14 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in elasticsearch-extensions.gemspec
 gemspec
 
-if File.exists? File.expand_path("../../elasticsearch-api/elasticsearch-api.gemspec", __FILE__)
+if File.exist? File.expand_path("../../elasticsearch-api/elasticsearch-api.gemspec", __FILE__)
   gem 'elasticsearch-api', :path => File.expand_path("../../elasticsearch-api", __FILE__), :require => false
 end
 
-if File.exists? File.expand_path("../../elasticsearch-transport/elasticsearch-transport.gemspec", __FILE__)
+if File.exist? File.expand_path("../../elasticsearch-transport/elasticsearch-transport.gemspec", __FILE__)
   gem 'elasticsearch-transport', :path => File.expand_path("../../elasticsearch-transport", __FILE__), :require => false
 end
 
-if File.exists? File.expand_path("../../elasticsearch-extensions", __FILE__)
-  gem 'elasticsearch-extensions', :path => File.expand_path("../../elasticsearch-extensions", __FILE__), :require => false
-end
-
-if File.exists? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__)
+if File.exist? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__)
   gem 'elasticsearch', :path => File.expand_path("../../elasticsearch/", __FILE__)
 end

data/README.md

@@ -136,10 +136,10 @@ Start the cluster on specific port, with a specific Elasticsearch version, numbe
     require 'elasticsearch/extensions/test/cluster'
 
     Elasticsearch::Extensions::Test::Cluster.start \
-      cluster_name: "my-testing-cluster",
-      command:      "/usr/local/Cellar/elasticsearch/0.90.10/bin/elasticsearch",
-      port:         9350,
-      nodes:        3
+      cluster_name:    "my-testing-cluster",
+      command:         "/usr/local/Cellar/elasticsearch/0.90.10/bin/elasticsearch",
+      port:            9350,
+      number_of_nodes: 3
 
     # Starting 3 Elasticsearch nodes.....................
     # --------------------------------------------------------------------------------

data/elasticsearch-extensions.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
 
   s.add_dependency "ansi"
 
-  unless File.exists? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__)
+  unless File.exist? File.expand_path("../../elasticsearch/elasticsearch.gemspec", __FILE__)
     s.add_dependency "elasticsearch"
   end
 
@@ -47,7 +47,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "ci_reporter", "~> 1.9"
 
   if defined?(RUBY_VERSION) && RUBY_VERSION < '1.9'
-    s.add_development_dependency "json"
+    s.add_development_dependency "json", '~> 1.8'
   end
 
   if defined?(RUBY_VERSION) && RUBY_VERSION > '1.9'

data/lib/elasticsearch/extensions/reindex.rb

@@ -125,28 +125,21 @@ module Elasticsearch
           response = arguments[:source][:client].search(
             index: arguments[:source][:index],
             scroll: arguments[:scroll],
-            size: arguments[:batch_size],
-            search_type: 'scan',
-            fields: ['_source', '_parent', '_routing', '_timestamp']
+            size: arguments[:batch_size]
           )
 
+          documents = response['hits']['hits']
+
+          unless documents.empty?
+            bulk_response = __store_batch(documents)
+            output[:errors] += bulk_response['items'].select { |k, v| k.values.first['error'] }.size
+          end
+
           while response = arguments[:source][:client].scroll(scroll_id: response['_scroll_id'], scroll: arguments[:scroll]) do
             documents = response['hits']['hits']
             break if documents.empty?
 
-            bulk = documents.map do |doc|
-              doc['_index'] = arguments[:target][:index]
-
-              arguments[:transform].call(doc) if arguments[:transform]
-
-              doc['data'] = doc['_source']
-              doc.delete('_score')
-              doc.delete('_source')
-
-              { index: doc }
-            end
-
-            bulk_response = arguments[:target][:client].bulk body: bulk
+            bulk_response = __store_batch(documents)
             output[:errors] += bulk_response['items'].select { |k, v| k.values.first['error'] }.size
           end
 
@@ -154,6 +147,22 @@ module Elasticsearch
 
           output
         end
+
+        def __store_batch(documents)
+          body = documents.map do |doc|
+            doc['_index'] = arguments[:target][:index]
+
+            arguments[:transform].call(doc) if arguments[:transform]
+
+            doc['data'] = doc['_source']
+            doc.delete('_score')
+            doc.delete('_source')
+
+            { index: doc }
+          end
+
+          arguments[:target][:client].bulk body: body
+        end
       end
     end
   end

data/lib/elasticsearch/extensions/test/cluster.rb

@@ -161,17 +161,20 @@ module Elasticsearch
                 -E path.repo=/tmp \
                 -E repositories.url.allowed_urls=http://snapshot.test* \
                 -E discovery.zen.minimum_master_nodes=#{arguments[:number_of_nodes]-1} \
+                -E node.max_local_storage_nodes=#{arguments[:number_of_nodes]} \
                 -E logger.level=DEBUG \
                 #{arguments[:es_params]} \
                 > /dev/null
               COMMAND
             }
           }
+          COMMANDS['6.0'] = COMMANDS['5.0']
+          COMMANDS.freeze
 
           # Create a new instance of the Cluster class
           #
           # @option arguments [String]  :cluster_name Cluster name (default: `elasticsearch_test`)
-          # @option arguments [Integer] :nodes        Number of desired nodes (default: 2)
+          # @option arguments [Integer] :number_of_nodes Number of desired nodes (default: 2)
           # @option arguments [String]  :command      Elasticsearch command (default: `elasticsearch`)
           # @option arguments [String]  :port         Starting port number; will be auto-incremented (default: 9250)
           # @option arguments [String]  :node_name    The node name (will be appended with a number)
@@ -188,7 +191,7 @@ module Elasticsearch
           # @see Cluster#start
           #
           def initialize(arguments={})
-            @arguments = arguments
+            @arguments = arguments.dup
 
             @arguments[:command]           ||= ENV.fetch('TEST_CLUSTER_COMMAND',   'elasticsearch')
             @arguments[:port]              ||= ENV.fetch('TEST_CLUSTER_PORT',      9250).to_i
@@ -221,7 +224,7 @@ module Elasticsearch
           # @example Start a cluster with a custom configuration
           #      Elasticsearch::Extensions::Test::Cluster::Cluster.new(
           #        cluster_name: 'my-cluster',
-          #        nodes: 3,
+          #        number_of_nodes: 3,
           #        node_name: 'my-node',
           #        port: 9350
           #      ).start
@@ -240,7 +243,7 @@ module Elasticsearch
               return false
             end
 
-            __remove_cluster_data
+            __remove_cluster_data if @clear_cluster
 
             STDOUT.print "Starting ".ansi(:faint) + arguments[:number_of_nodes].to_s.ansi(:bold, :faint) +
                          " Elasticsearch nodes..".ansi(:faint)
@@ -256,6 +259,7 @@ module Elasticsearch
               pid = Process.spawn(command)
               Process.detach pid
               pids << pid
+              sleep 1
             end
 
             __check_for_running_processes(pids)
@@ -365,8 +369,8 @@ module Elasticsearch
               when /^0|^1/
                 '0.0.0.0'
               when /^2/
-                '0.0.0.0'
-              when /^5/
+                '_local_'
+              when /^5|^6/
                 '_local_'
               else
                 raise RuntimeError, "Cannot determine default network host from version [#{version}]"
@@ -418,18 +422,34 @@ module Elasticsearch
                 raise RuntimeError, "Cannot determine Elasticsearch version from jar [#{jar}]"
               end
             else
-              STDERR.puts "[!] Cannot find Elasticsearch .jar from path to command [#{arguments[:command]}], using `elasticsearch --version`" if ENV['DEBUG']
+              STDERR.puts "[!] Cannot find Elasticsearch .jar from path to command [#{arguments[:command]}], using `#{arguments[:command]} --version`" if ENV['DEBUG']
+
+              unless File.exist? arguments[:command]
+                raise Errno::ENOENT, "File [#{arguments[:command]}] does not exist -- did you pass a correct path to the Elasticsearch launch script"
+              end
 
               output = ''
 
               begin
                 # First, try the new `--version` syntax...
                 STDERR.puts "Running [#{arguments[:command]} --version] to determine version" if ENV['DEBUG']
-                Timeout::timeout(10) { output = `#{arguments[:command]} --version` }
+                rout, wout = IO.pipe
+                pid = Process.spawn("#{arguments[:command]} --version", out: wout)
+
+                Timeout::timeout(10) do
+                  Process.wait(pid)
+                  wout.close unless wout.closed?
+                  output = rout.read unless rout.closed?
+                  rout.close unless rout.closed?
+                end
               rescue Timeout::Error
-                # ...else, the new `-v` syntax
+                # ...else, the old `-v` syntax
                 STDERR.puts "Running [#{arguments[:command]} -v] to determine version" if ENV['DEBUG']
                 output = `#{arguments[:command]} -v`
+              ensure
+                Process.kill('INT', pid) if pid
+                wout.close unless wout.closed?
+                rout.close unless rout.closed?
               end
 
               STDERR.puts "> #{output}" if ENV['DEBUG']
@@ -454,6 +474,8 @@ module Elasticsearch
                 '2.0'
               when /^5\..*/
                 '5.0'
+              when /^6\..*/
+                '6.0'
               else
                 raise RuntimeError, "Cannot determine major version from [#{version}]"
             end
@@ -485,17 +507,25 @@ module Elasticsearch
           # @return Boolean
           #
           def __wait_for_status(status='green', timeout=30)
-            Timeout::timeout(timeout) do
-              loop do
-                response = __get_cluster_health(status)
+            begin
+              Timeout::timeout(timeout) do
+                loop do
+                  response = __get_cluster_health(status)
+                  STDERR.puts response if ENV['DEBUG']
 
-                if response && response['status'] == status && ( arguments[:number_of_nodes].nil? || arguments[:number_of_nodes].to_i == response['number_of_nodes'].to_i  )
-                  break
-                end
+                  if response && response['status'] == status && ( arguments[:number_of_nodes].nil? || arguments[:number_of_nodes].to_i == response['number_of_nodes'].to_i  )
+                    break
+                  end
 
-                STDOUT.print '.'.ansi(:faint)
-                sleep 1
+                  STDOUT.print '.'.ansi(:faint)
+                  sleep 1
+                end
               end
+            rescue Timeout::Error => e
+              message = "\nTimeout while waiting for cluster status [#{status}]"
+              message += " and [#{arguments[:number_of_nodes]}] nodes" if arguments[:number_of_nodes]
+              STDOUT.puts message.ansi(:red, :bold)
+              raise e
             end
 
             return true
@@ -553,13 +583,12 @@ module Elasticsearch
             JSON.parse(response)
           end
 
-          # Remove the data directory (unless it has been disabled by arguments)
+          # Remove the data directory
           #
           # @api private
           #
           def __remove_cluster_data
-            # Wipe out data on disk for this cluster name by default
-            FileUtils.rm_rf "#{arguments[:path_data]}/#{arguments[:cluster_name]}" if @clear_cluster
+            FileUtils.rm_rf arguments[:path_data]
           end
 
 

data/lib/elasticsearch/extensions/version.rb

@@ -1,5 +1,5 @@
 module Elasticsearch
   module Extensions
-    VERSION = "0.0.22"
+    VERSION = "0.0.23"
   end
 end

data/test/reindex/integration/reindex_test.rb

@@ -25,13 +25,20 @@ class Elasticsearch::Extensions::ReindexIntegrationTest < Elasticsearch::Test::I
       @client.index index: 'test1', type: 'd', id: 3, body: { title: 'TEST 3', category: 'three' }
       @client.indices.refresh index: 'test1'
 
+      @client.indices.create index: 'test2'
+
       @client.cluster.health wait_for_status: 'yellow'
     end
 
+    teardown do
+      @client.indices.delete index: '_all'
+    end
+
     should "copy documents from one index to another" do
       reindex = Elasticsearch::Extensions::Reindex.new \
                   source: { index: 'test1', client: @client },
                   target: { index: 'test2' },
+                  batch_size: 2,
                   refresh: true
 
       result = reindex.perform
@@ -40,6 +47,36 @@ class Elasticsearch::Extensions::ReindexIntegrationTest < Elasticsearch::Test::I
       assert_equal 3, @client.search(index: 'test2')['hits']['total']
     end
 
+    should "copy documents with parent/child relationship" do
+      mapping = { mappings: { p: {}, c: { _parent: { type: 'p' } } } }
+      @client.indices.create index: 'test_parent_1', body: mapping
+      @client.indices.create index: 'test_parent_2', body: mapping
+
+      @client.index index: 'test_parent_1', type: 'p', id: 1, body: { title: 'Parent 1' }
+      @client.index index: 'test_parent_1', type: 'p', id: 2, body: { title: 'Parent 2' }
+      @client.index index: 'test_parent_1', type: 'c', parent: 1, body: { title: 'Child One' }
+      @client.index index: 'test_parent_1', type: 'c', parent: 1, body: { title: 'Child Two' }
+
+      @client.indices.refresh index: 'test_parent_1'
+
+      reindex = Elasticsearch::Extensions::Reindex.new \
+                  source: { index: 'test_parent_1', client: @client },
+                  target: { index: 'test_parent_2' },
+                  batch_size: 2,
+                  refresh: true
+
+      result = reindex.perform
+
+      assert_equal 0, result[:errors]
+      assert_equal 4, @client.search(index: 'test_parent_2')['hits']['total']
+
+      response = @client.search index: 'test_parent_2', body: {
+        query: { has_child: { type: 'c', query: { match: { title: 'two' } } } } }
+
+      assert_equal 1, response['hits']['hits'].size
+      assert_equal 'Parent 1', response['hits']['hits'][0]['_source']['title']
+    end
+
     should "transform documents with a lambda" do
       reindex = Elasticsearch::Extensions::Reindex.new \
                   source: { index: 'test1', client: @client },
@@ -60,18 +97,20 @@ class Elasticsearch::Extensions::ReindexIntegrationTest < Elasticsearch::Test::I
 
       reindex = Elasticsearch::Extensions::Reindex.new \
                   source: { index: 'test1', client: @client },
-                  target: { index: 'test3', transform: lambda { |d| d['_source']['category'].upcase!; d } },
-                  refresh: true
+                  target: { index: 'test3' }
 
       result = reindex.perform
 
+      @client.indices.refresh index: 'test3'
+
       assert_equal 3, result[:errors]
       assert_equal 0, @client.search(index: 'test3')['hits']['total']
     end
 
     should "reindex via the API integration" do
-      @client.reindex source: { index: 'test1' }, target: { index: 'test4' }
+      @client.indices.create index: 'test4'
 
+      @client.reindex source: { index: 'test1' }, target: { index: 'test4' }
       @client.indices.refresh index: 'test4'
 
       assert_equal 3, @client.search(index: 'test4')['hits']['total']

data/test/reindex/unit/reindex_test.rb

@@ -43,8 +43,8 @@ class Elasticsearch::Extensions::ReindexTest < Test::Unit::TestCase
                                 '_id'    => d['_id'],
                                 'data'   => d['_source']
                                } }]
-        @bulk_response    = {'errors'=>false, 'items' => [{'index' => {}}]}
-        @bulk_response_error = {'errors'=>true, 'items' => [{'index' => {}},{'index' => {'error' => 'FOOBAR'}}]}
+        @bulk_response    = {'errors'=>false, 'items' => [{'index' => {}}, {'index' => {}}]}
+        @bulk_response_error = {'errors'=>true, 'items' => [{'index' => {}}, {'index' => {'error' => 'FOOBAR'}}]}
       end
 
       should "scroll through the index and save batches in bulk" do
@@ -52,11 +52,15 @@ class Elasticsearch::Extensions::ReindexTest < Test::Unit::TestCase
         subject = Elasticsearch::Extensions::Reindex.new source: { index: 'foo', client: client },
                                                          target: { index: 'bar' }
 
-        client.expects(:search).returns({ '_scroll_id' => 'scroll_id_1' })
-        client.expects(:scroll).returns(@default_response)
-              .then.returns(@empty_response)
-              .times(2)
-        client.expects(:bulk).with(body: @bulk_request).returns(@bulk_response)
+        client.expects(:search)
+          .returns({ '_scroll_id' => 'scroll_id_1' }.merge(Marshal.load(Marshal.dump(@default_response))))
+        client.expects(:scroll)
+          .returns(Marshal.load(Marshal.dump(@default_response)))
+          .then
+          .returns(@empty_response).times(2)
+        client.expects(:bulk)
+          .with(body: @bulk_request)
+          .returns(@bulk_response).times(2)
 
         result = subject.perform
 
@@ -68,10 +72,8 @@ class Elasticsearch::Extensions::ReindexTest < Test::Unit::TestCase
         subject = Elasticsearch::Extensions::Reindex.new source: { index: 'foo', client: client },
                                                          target: { index: 'bar' }
 
-        client.expects(:search).returns({ '_scroll_id' => 'scroll_id_1' })
-        client.expects(:scroll).returns(@default_response)
-              .then.returns(@empty_response)
-              .times(2)
+        client.expects(:search).returns({ '_scroll_id' => 'scroll_id_1' }.merge(@default_response))
+        client.expects(:scroll).returns(@empty_response)
         client.expects(:bulk).with(body: @bulk_request).returns(@bulk_response_error)
 
         result = subject.perform
@@ -86,10 +88,8 @@ class Elasticsearch::Extensions::ReindexTest < Test::Unit::TestCase
           target: { index: 'bar' },
           transform: lambda { |d| d['_source']['foo'].upcase!; d }
 
-        client.expects(:search).returns({ '_scroll_id' => 'scroll_id_1' })
-        client.expects(:scroll).returns(@default_response)
-              .then.returns(@empty_response)
-              .times(2)
+        client.expects(:search).returns({ '_scroll_id' => 'scroll_id_1' }.merge(@default_response))
+        client.expects(:scroll).returns(@empty_response)
         client.expects(:bulk).with do |arguments|
                 assert_equal 'BAR', arguments[:body][0][:index]['data']['foo']
                 true

data/test/test/cluster/integration/cluster_test.rb

@@ -22,6 +22,11 @@ class Elasticsearch::Extensions::TestClusterIntegrationTest < Test::Unit::TestCa
       should "start and stop #{build.to_s}" do
         puts ("----- #{build.to_s} " + "-"*(80-7-build.to_s.size)).to_s.ansi(:bold)
         Elasticsearch::Extensions::Test::Cluster.start command: PATH_TO_BUILDS.join(build.join('bin/elasticsearch')).to_s
+
+        # Index some data to create the data directory
+        client = Elasticsearch::Client.new host: "localhost:9250"
+        client.index index: 'test1', type: 'd', id: 1, body: { title: 'TEST' }
+
         Elasticsearch::Extensions::Test::Cluster.stop command: PATH_TO_BUILDS.join(build.join('bin/elasticsearch')).to_s
       end
     end

data/test/test/cluster/unit/cluster_test.rb

@@ -38,6 +38,13 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase
         assert_equal 9400, c.arguments[:port]
       end
 
+      should "not modify the arguments" do
+        args = { port: 9400 }.freeze
+
+        assert_nothing_raised { Cluster::Cluster.new args }
+        assert_nil args[:command]
+      end
+
       should "take parameters from environment variables" do
         ENV['TEST_CLUSTER_PORT'] = '9400'
 
@@ -88,6 +95,13 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase
         end
       end
 
+      should "remove cluster data" do
+        @subject.unstub(:__remove_cluster_data)
+        FileUtils.expects(:rm_rf).with("/tmp/elasticsearch_test")
+
+        @subject.__remove_cluster_data
+      end
+
       context "when starting a cluster, " do
         should "return false when it's already running" do
           Process.expects(:spawn).never
@@ -238,11 +252,18 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase
           assert_equal '2.0', @subject.__determine_version
         end
 
-        should "return version from `elasticsearch -v`" do
+        should "return version from `elasticsearch --version`" do
           File.expects(:exist?).with('/foo/bar/bin/../lib/').returns(false)
 
-          @subject.expects(:`)
-            .with("/foo/bar/bin/elasticsearch --version")
+          Process.stubs(:wait)
+          Process.expects(:spawn)
+            .with do |command, options|
+              assert_equal "/foo/bar/bin/elasticsearch --version", command
+            end
+            .returns(123)
+          Process.expects(:kill).with('INT', 123)
+
+          IO.any_instance.expects(:read)
             .returns('Version: 2.3.0-SNAPSHOT, Build: d1c86b0/2016-03-30T10:43:20Z, JVM: 1.8.0_60')
 
           assert_equal '2.0', @subject.__determine_version
@@ -259,9 +280,15 @@ class Elasticsearch::Extensions::TestClusterTest < Test::Unit::TestCase
         should "raise an exception when the version cannot be parsed from command output" do
           File.expects(:exist?).with('/foo/bar/bin/../lib/').returns(false)
 
-          @subject.expects(:`)
-            .with("/foo/bar/bin/elasticsearch --version")
-            .returns('Version: FOOBAR')
+          Process.stubs(:wait)
+          Process.expects(:spawn)
+            .with do |command, options|
+              assert_equal "/foo/bar/bin/elasticsearch --version", command
+            end
+            .returns(123)
+          Process.expects(:kill).with('INT', 123)
+
+          IO.any_instance.expects(:read).returns('Version: FOOBAR')
 
           assert_raise(RuntimeError) { @subject.__determine_version }
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-extensions
 version: !ruby/object:Gem::Version
-  version: 0.0.22
+  version: 0.0.23
 platform: ruby
 authors:
 - Karel Minarik
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-21 00:00:00.000000000 Z
+date: 2016-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ansi
@@ -313,4 +313,3 @@ test_files:
 - test/test/cluster/integration/cluster_test.rb
 - test/test/cluster/unit/cluster_test.rb
 - test/test_helper.rb
-has_rdoc: