elasticsearch-extensions 0.0.27 → 0.0.32

data/lib/elasticsearch/extensions.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 require 'elasticsearch'

data/lib/elasticsearch/extensions/ansi.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 require 'elasticsearch/extensions'

data/lib/elasticsearch/extensions/ansi/actions.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 module Elasticsearch
@@ -100,7 +117,7 @@ module Elasticsearch
             output << ""
           end
           output << Helpers.___
-          output << "#{hits.size.to_s.ansi(:bold)} of #{json['hits']['total'].to_s.ansi(:bold)} results".ansi(:faint)
+          output << "#{hits.size.to_s.ansi(:bold)} of #{json['hits']['total']['value'].to_s.ansi(:bold)} results".ansi(:faint)
 
           output.join("\n")
         end

data/lib/elasticsearch/extensions/ansi/helpers.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 module Elasticsearch

data/lib/elasticsearch/extensions/ansi/response.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 module Elasticsearch
   module Extensions
     module ANSI

data/lib/elasticsearch/extensions/backup.rb

@@ -1,13 +1,41 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 require 'pathname'
 require 'fileutils'
 
 require 'multi_json'
-require 'oj'
+
+begin
+  require 'oj'
+rescue LoadError
+  warn('The "oj" gem could not be loaded. JSON parsing and serialization performance may not be optimal.')
+end
+
+begin
+  require 'patron'
+rescue LoadError
+  warn('The "patron" gem could not be loaded. HTTP requests may not be performed optimally.')
+end
+
 
 require 'elasticsearch'
-require 'patron'
 
 module Backup
   module Database
@@ -154,12 +182,19 @@ module Backup
         while r = client.scroll(scroll_id: r['_scroll_id'], scroll: scroll) and not r['hits']['hits'].empty? do
           r['hits']['hits'].each do |hit|
             FileUtils.mkdir_p "#{path.join hit['_index'], hit['_type']}"
-            File.open("#{path.join hit['_index'], hit['_type'], hit['_id']}.json", 'w') do |file|
+            File.open("#{path.join hit['_index'], hit['_type'], __sanitize_filename(hit['_id'])}.json", 'w') do |file|
               file.write MultiJson.dump(hit)
             end
           end
         end
       end
+
+      def __sanitize_filename name
+        name
+          .encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�")
+          .strip
+          .tr("\u{202E}%$|:;/\t\r\n\\", "-")
+      end
     end
   end
 end

data/lib/elasticsearch/extensions/reindex.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 module Elasticsearch
@@ -23,10 +40,10 @@ module Elasticsearch
       end; extend self
 
       module API
-        # Copy documents from one index into another and refresh the target index
+        # Copy documents from one index into another and refresh the destination index
         #
         # @example
-        #     client.reindex source: { index: 'test1' }, target: { index: 'test2' }, refresh: true
+        #     client.reindex source: { index: 'test1' }, dest: { index: 'test2' }, refresh: true
         #
         # The method allows all the options as {Reindex::Reindex.new}.
         #
@@ -50,40 +67,40 @@ module Elasticsearch
       #   client  = Elasticsearch::Client.new
       #   reindex = Elasticsearch::Extensions::Reindex.new \
       #               source: { index: 'test1', client: client },
-      #               target: { index: 'test2' }
+      #               dest: { index: 'test2' }
       #
       #   reindex.perform
       #
       # @example Copy documents to a different cluster
       #
       #     source_client  = Elasticsearch::Client.new url: 'http://localhost:9200'
-      #     target_client  = Elasticsearch::Client.new url: 'http://localhost:9250'
+      #     destination_client  = Elasticsearch::Client.new url: 'http://localhost:9250'
       #
       #     reindex = Elasticsearch::Extensions::Reindex.new \
       #                 source: { index: 'test', client: source_client },
-      #                 target: { index: 'test', client: target_client }
+      #                 dest: { index: 'test', client: destination_client }
       #     reindex.perform
       #
       # @example Transform the documents during re-indexing
       #
       #     reindex = Elasticsearch::Extensions::Reindex.new \
       #                 source: { index: 'test1', client: client },
-      #                 target: { index: 'test2' },
+      #                 dest: { index: 'test2' },
       #                 transform: lambda { |doc| doc['_source']['category'].upcase! }
       #
       #
       # The reindexing process works by "scrolling" an index and sending
-      # batches via the "Bulk" API to the target index/cluster
+      # batches via the "Bulk" API to the destination index/cluster
       #
       # @option arguments [String] :source The source index/cluster definition (*Required*)
-      # @option arguments [String] :target The target index/cluster definition (*Required*)
+      # @option arguments [String] :dest The destination index/cluster definition (*Required*)
       # @option arguments [Proc] :transform A block which will be executed for each document
       # @option arguments [Integer] :batch_size The size of the batch for scroll operation (Default: 1000)
       # @option arguments [String] :scroll The timeout for the scroll operation (Default: 5min)
-      # @option arguments [Boolean] :refresh Whether to refresh the target index after
+      # @option arguments [Boolean] :refresh Whether to refresh the destination index after
       #                                      the operation is completed (Default: false)
       #
-      # Be aware, that if you want to change the target index settings and/or mappings,
+      # Be aware, that if you want to change the destination index settings and/or mappings,
       # you have to do so in advance by using the "Indices Create" API.
       #
       # Note, that there is a native "Reindex" API in Elasticsearch 2.3.x and higer versions,
@@ -98,7 +115,7 @@ module Elasticsearch
           [
             [:source, :index],
             [:source, :client],
-            [:target, :index]
+            [:dest, :index]
           ].each do |required_option|
             value = required_option.reduce(arguments) { |sum, o| sum = sum[o] ? sum[o] : {}  }
 
@@ -113,7 +130,7 @@ module Elasticsearch
             refresh: false
           }.merge(arguments)
 
-          arguments[:target][:client] ||= arguments[:source][:client]
+          arguments[:dest][:client] ||= arguments[:source][:client]
         end
 
         # Performs the operation
@@ -144,14 +161,14 @@ module Elasticsearch
             output[:errors] += bulk_response['items'].select { |k, v| k.values.first['error'] }.size
           end
 
-          arguments[:target][:client].indices.refresh index: arguments[:target][:index] if arguments[:refresh]
+          arguments[:dest][:client].indices.refresh index: arguments[:dest][:index] if arguments[:refresh]
 
           output
         end
 
         def __store_batch(documents)
           body = documents.map do |doc|
-            doc['_index'] = arguments[:target][:index]
+            doc['_index'] = arguments[:dest][:index]
 
             arguments[:transform].call(doc) if arguments[:transform]
 
@@ -162,7 +179,7 @@ module Elasticsearch
             { index: doc }
           end
 
-          arguments[:target][:client].bulk body: body
+          arguments[:dest][:client].bulk body: body
         end
       end
     end

data/lib/elasticsearch/extensions/test/cluster.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 require 'timeout'
 require 'net/http'
 require 'fileutils'
@@ -21,7 +38,6 @@ end
 module Elasticsearch
   module Extensions
     module Test
-
       # A convenience Ruby class for starting and stopping an Elasticsearch cluster,
       # eg. for integration tests
       #
@@ -43,12 +59,11 @@ module Elasticsearch
       # @see Cluster#initialize
       #
       module Cluster
-
         # Starts a cluster
         #
         # @see Cluster#start
         #
-        def start(arguments={})
+        def start(arguments = {})
           Cluster.new(arguments).start
         end
 
@@ -56,7 +71,7 @@ module Elasticsearch
         #
         # @see Cluster#stop
         #
-        def stop(arguments={})
+        def stop(arguments = {})
           Cluster.new(arguments).stop
         end
 
@@ -64,7 +79,7 @@ module Elasticsearch
         #
         # @see Cluster#running?
         #
-        def running?(arguments={})
+        def running?(arguments = {})
           Cluster.new(arguments).running?
         end
 
@@ -72,7 +87,7 @@ module Elasticsearch
         #
         # @see Cluster#wait_for_green
         #
-        def wait_for_green(arguments={})
+        def wait_for_green(arguments = {})
           Cluster.new(arguments).wait_for_green
         end
 
@@ -190,12 +205,15 @@ module Elasticsearch
                 -E path.repo=/tmp \
                 -E repositories.url.allowed_urls=http://snapshot.test* \
                 -E discovery.zen.minimum_master_nodes=#{arguments[:number_of_nodes]-1} \
+                -E xpack.security.enabled=false \
                 -E node.max_local_storage_nodes=#{arguments[:number_of_nodes]} \
                 -E logger.level=#{ENV['DEBUG'] ? 'DEBUG' : 'INFO'} \
                 #{arguments[:es_params]}
               COMMAND
             }
           }
+          COMMANDS['7.0'] = COMMANDS['6.0'].clone
+          COMMANDS['8.0'] = COMMANDS['7.0'].clone
           COMMANDS.freeze
 
           # Create a new instance of the Cluster class
@@ -237,7 +255,11 @@ module Elasticsearch
             @arguments[:network_host]      ||= ENV.fetch('TEST_CLUSTER_NETWORK_HOST', __default_network_host)
             @arguments[:quiet]             ||= ! ENV.fetch('QUIET', '').empty?
 
-            @clear_cluster = !!@arguments[:clear_cluster] || (ENV.fetch('TEST_CLUSTER_CLEAR', 'true') != 'false')
+            @clear_cluster = if @arguments[:clear_cluster].nil?
+                               (ENV.fetch('TEST_CLUSTER_CLEAR', 'true') != 'false')
+                             else
+		                           !!@arguments[:clear_cluster]
+                             end
 
             # Make sure `cluster_name` is not dangerous
             raise ArgumentError, "The `cluster_name` argument cannot be empty string or a slash" \
@@ -407,7 +429,7 @@ module Elasticsearch
                 '0.0.0.0'
               when /^2/
                 '_local_'
-              when /^5|^6/
+              when /^5|^6|^7|^8/
                 '_local_'
               else
                 raise RuntimeError, "Cannot determine default network host from version [#{version}]"
@@ -450,9 +472,10 @@ module Elasticsearch
           #
           def __determine_version
             path_to_lib = File.dirname(arguments[:command]) + '/../lib/'
+
             version = if arguments[:version]
               arguments[:version]
-            elsif File.exist?(path_to_lib) && !(jar = Dir.entries(path_to_lib).select { |f| f.start_with? 'elasticsearch' }.first).nil?
+            elsif File.exist?(path_to_lib) && !(jar = Dir.entries(path_to_lib).select { |f| f =~ /^elasticsearch\-\d/ }.first).nil?
               __log "Determining version from [#{jar}]" if ENV['DEBUG']
               if m = jar.match(/elasticsearch\-(\d+\.\d+\.\d+).*/)
                 m[1]
@@ -482,15 +505,14 @@ module Elasticsearch
               begin
                 # First, try the new `--version` syntax...
                 __log "Running [#{arguments[:command]} --version] to determine version" if ENV['DEBUG']
-                rout, wout = IO.pipe
-                pid = Process.spawn("#{arguments[:command]} --version", out: wout)
+                io = IO.popen("#{arguments[:command]} --version")
+                pid = io.pid
 
                 Timeout::timeout(arguments[:timeout_version]) do
                   Process.wait(pid)
-                  wout.close unless wout.closed?
-                  output = rout.read unless rout.closed?
-                  rout.close unless rout.closed?
+                  output = io.read
                 end
+
               rescue Timeout::Error
                 # ...else, the old `-v` syntax
                 __log "Running [#{arguments[:command]} -v] to determine version" if ENV['DEBUG']
@@ -499,8 +521,7 @@ module Elasticsearch
                 if pid
                   Process.kill('INT', pid) rescue Errno::ESRCH # Most likely the process has terminated already
                 end
-                wout.close unless wout.closed?
-                rout.close unless rout.closed?
+                io.close unless io.closed?
               end
 
               STDERR.puts "> #{output}" if ENV['DEBUG']
@@ -509,7 +530,7 @@ module Elasticsearch
                 raise RuntimeError, "Cannot determine Elasticsearch version from [#{arguments[:command]} --version] or [#{arguments[:command]} -v]"
               end
 
-              if m = output.match(/Version: (\d\.\d.\d).*,/)
+              if(m = output.match(/Version: (\d+\.\d+.\d+).*,/))
                 m[1]
               else
                 raise RuntimeError, "Cannot determine Elasticsearch version from elasticsearch --version output [#{output}]"
@@ -527,6 +548,10 @@ module Elasticsearch
                 '5.0'
               when /^6\..*/
                 '6.0'
+              when /^7\..*/
+                '7.0'
+              when /^8\..*/
+                '8.0'
               else
                 raise RuntimeError, "Cannot determine major version from [#{version}]"
             end