elasticsearch 8.8.0 → 8.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c72b5eb32153d9598565613a720c140bef43a1794cebcd5f9f75f977a96700ef
-  data.tar.gz: 07b1eb4c84634993423f26519d87115c84e70f8a67f952469149209169a4701a
+  metadata.gz: 2e2183bdba09aed6383309a842c989713effa04d48af7385b43bcb80e2c01b1b
+  data.tar.gz: f5b9971c94be976c32a4bb7f298832659b6da4db9f11aaf5d5f1deb5878110f6
 SHA512:
-  metadata.gz: 3e0ff385f6834e5a5c8f07db48caaf65e9eb08797f307dc8b9e6869b2202006bbdcb6cd64c9d9d3d4dd4c735340e18aaa019a44f2511ae8b34d07dba74106f41
-  data.tar.gz: 7b2bed8e82f2c1e421951910143b7455567d6c78dc74775c8502233422c2d20bd6c8249a03d543f82be379dfaab498085f6ae7fab27745788456f68eb0940158
+  metadata.gz: da2903828837d1286fa486f8f2e8458b4cf82b44a885f1742976527159f22291aa9d6a9ccbd1d7e6011222f68ec11fe26392f25d3357cc08b4c967f6c83d0d6f
+  data.tar.gz: c95be1ee39a3d182e085a50d34de1d0650ef170333e1405c1161e28c0153f5a1e3f9349cedecd62e3f1598776e510436477c7583f879ed11af77f9d76a3dc5ca

data/bin/elastic_ruby_console

@@ -2,9 +2,11 @@
 
 $LOAD_PATH.unshift(File.expand_path('../../elasticsearch/lib', __dir__))
 $LOAD_PATH.unshift(File.expand_path('../../elasticsearch-api/lib', __dir__))
+$LOAD_PATH.unshift(File.expand_path('../../elasticsearch/lib/elasticsearch/helpers', __dir__))
 
 require 'elasticsearch'
 require 'elasticsearch-api'
+require 'elasticsearch/helpers/bulk_helper'
 
 include Elasticsearch
 

data/elasticsearch.gemspec

@@ -46,7 +46,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.5'
 
   s.add_dependency 'elastic-transport', '~> 8'
-  s.add_dependency 'elasticsearch-api', '8.8.0'
+  s.add_dependency 'elasticsearch-api', '8.9.0'
 
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'byebug' unless defined?(JRUBY_VERSION) || defined?(Rubinius)

data/lib/elasticsearch/helpers/bulk_helper.rb

@@ -0,0 +1,127 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Elasticsearch
+  module Helpers
+    # Elasticsearch Client Helper for the Bulk API
+    #
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-bulk.html
+    #
+    class BulkHelper
+      attr_accessor :index
+
+      # Create a BulkHelper
+      #
+      # @param [Elasticsearch::Client] client Instance of Elasticsearch client to use.
+      # @param [String] index Index on which to perform the Bulk actions.
+      # @param [Hash] params Parameters to re-use in every bulk call
+      #
+      def initialize(client, index, params = {})
+        @client = client
+        @index = index
+        @params = params
+      end
+
+      # Index documents using the Bulk API.
+      #
+      # @param [Array<Hash>] docs The documents to be indexed.
+      # @param [Hash] params Parameters to use in the bulk ingestion. See the official Elastic documentation for Bulk API for parameters to send to the Bulk API.
+      # @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of ingestion.
+      # @param block [Block] Optional block to run after ingesting a batch of documents.
+      # @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
+      # @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
+      #
+      def ingest(docs, params = {}, body = {}, &block)
+        ingest_docs = docs.map { |doc| { index: { _index: @index, data: doc} } }
+        if (slice = params.delete(:slice))
+          ingest_docs.each_slice(slice) { |items| ingest(items, params, &block) }
+        else
+          bulk_request(ingest_docs, params, &block)
+        end
+      end
+
+      # Delete documents using the Bulk API
+      #
+      # @param [Array] ids Array of id's for documents to delete.
+      # @param [Hash] params Parameters to send to bulk delete.
+      #
+      def delete(ids, params = {}, body = {})
+        delete_docs = ids.map { |id| { delete: { _index: @index, _id: id} } }
+        @client.bulk({ body: delete_docs }.merge(params.merge(@params)))
+      end
+
+      # Update documents using the Bulk API
+      #
+      # @param [Array<Hash>] docs (Required) The documents to be updated.
+      # @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of updates.
+      # @param block [Block]  Optional block to run after ingesting a batch of documents.
+      #
+      # @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
+      # @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
+      #
+      def update(docs, params = {}, body = {}, &block)
+        ingest_docs = docs.map do |doc|
+          { update: { _index: @index, _id: doc.delete('id'), data: { doc: doc } } }
+        end
+        if (slice = params.delete(:slice))
+          ingest_docs.each_slice(slice) { |items| update(items, params, &block) }
+        else
+          bulk_request(ingest_docs, params, &block)
+        end
+      end
+
+      # Ingest data directly from a JSON file
+      #
+      # @param [String] file (Required) The file path.
+      # @param [Hash] params Parameters to use in the bulk ingestion.
+      # @option params [Integer] slice number of documents to send to the Bulk API for eatch batch of updates.
+      # @option params [Array|String] keys If the data needs to be digged from the JSON file, the
+      #                                      keys can be passed in  with this parameter to find it.
+      #
+      #                                      E.g.: If the data in the parsed JSON Hash is found in
+      #                                      +json_parsed['data']['items']+, keys would be passed
+      #                                      like this (as an Array):
+      #
+      #                                      +bulk_helper.ingest_json(file, { keys: ['data', 'items'] })+
+      #
+      #                                      or as a String:
+      #
+      #                                      +bulk_helper.ingest_json(file, { keys: 'data, items' })+
+      #
+      # @yieldparam response [Elasticsearch::Transport::Response] The response object from calling the Bulk API.
+      # @yieldparam ingest_docs [Array<Hash>] The collection of documents sent in the bulk request.
+      #
+      def ingest_json(file, params = {}, &block)
+        data = JSON.parse(File.read(file))
+        if (keys = params.delete(:keys))
+          keys = keys.split(',') if keys.is_a?(String)
+          data = data.dig(*keys)
+        end
+
+        ingest(data, params, &block)
+      end
+
+      private
+
+      def bulk_request(ingest_docs, params, &block)
+        response = @client.bulk({ body: ingest_docs }.merge(params.merge(@params)))
+        yield response, ingest_docs if block_given?
+        response
+      end
+    end
+  end
+end

data/lib/elasticsearch/helpers/scroll_helper.rb

@@ -0,0 +1,95 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Elasticsearch
+  module Helpers
+    # Elasticsearch Client Helper for the Scroll API
+    #
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/scroll-api.html
+    #
+    class ScrollHelper
+      include Enumerable
+
+      # Create a ScrollHelper
+      #
+      # @param [Elasticsearch::Client] client (Required) Instance of Elasticsearch client to use.
+      # @param [String] index (Required) Index on which to perform the Bulk actions.
+      # @param [Hash] body Body parameters to re-use in every scroll request
+      # @param [Time] scroll Specify how long a consistent view of the index should be maintained for scrolled search
+      #
+      def initialize(client, index, body, scroll = '1m')
+        @index = index
+        @client = client
+        @scroll = scroll
+        @body = body
+      end
+
+      # Implementation of +each+ for Enumerable module inclusion
+      #
+      # @yieldparam document [Hash] yields a document found in the search hits.
+      #
+      def each(&block)
+        @docs = []
+        @scroll_id = nil
+        refresh_docs
+        for doc in @docs do
+          refresh_docs
+          yield doc
+        end
+        clear
+      end
+
+      # Results from a scroll.
+      # Can be called repeatedly (e.g. in a loop) to get the scroll pages.
+      #
+      def results
+        if @scroll_id
+          scroll_request
+        else
+          initial_search
+        end
+      rescue StandardError => e
+        raise e
+      end
+
+      # Clear Scroll and resets inner documents collection
+      #
+      def clear
+        @client.clear_scroll(body: { scroll_id: @scroll_id }) if @scroll_id
+        @docs = []
+      end
+
+      private
+
+      def refresh_docs
+        @docs ||= []
+        @docs << results
+        @docs.flatten!
+      end
+
+      def initial_search
+        response = @client.search(index: @index, scroll: @scroll, body: @body)
+        @scroll_id = response['_scroll_id']
+        response['hits']['hits']
+      end
+
+      def scroll_request
+        @client.scroll(body: {scroll: @scroll, scroll_id: @scroll_id})['hits']['hits']
+      end
+    end
+  end
+end

data/lib/elasticsearch/version.rb

@@ -16,5 +16,5 @@
 # under the License.
 
 module Elasticsearch
-  VERSION = '8.8.0'.freeze
+  VERSION = '8.9.0'.freeze
 end

data/lib/elasticsearch.rb

@@ -35,24 +35,22 @@ module Elasticsearch
 
     # Create a client connected to an Elasticsearch cluster.
     #
+    # @param [Hash] arguments - initializer arguments
     # @option arguments [String] :cloud_id - The Cloud ID to connect to Elastic Cloud
-    # @option api_key [String, Hash] :api_key Use API Key Authentication, either the base64 encoding of `id` and `api_key`
-    #                                         joined by a colon as a String, or a hash with the `id` and `api_key` values.
-    # @option opaque_id_prefix [String] :opaque_id_prefix set a prefix for X-Opaque-Id when initializing the client.
-    #                                                     This will be prepended to the id you set before each request
-    #                                                     if you're using X-Opaque-Id
+    # @option arguments [String, Hash] :api_key Use API Key Authentication, either the base64 encoding of `id` and `api_key`
+    #                                           joined by a colon as a String, or a hash with the `id` and `api_key` values.
+    # @option arguments [String] :opaque_id_prefix set a prefix for X-Opaque-Id when initializing the client.
+    #                                              This will be prepended to the id you set before each request
+    #                                              if you're using X-Opaque-Id
+    # @option arguments [Hash] :headers Custom HTTP Request Headers
+    #
     def initialize(arguments = {}, &block)
       @verified = false
+      @warned = false
       @opaque_id_prefix = arguments[:opaque_id_prefix] || nil
       api_key(arguments) if arguments[:api_key]
-      if arguments[:cloud_id]
-        arguments[:hosts] = setup_cloud_host(
-          arguments[:cloud_id],
-          arguments[:user],
-          arguments[:password],
-          arguments[:port]
-        )
-      end
+      setup_cloud(arguments) if arguments[:cloud_id]
+      set_user_agent!(arguments) unless sent_user_agent?(arguments)
       @transport = Elastic::Transport::Client.new(arguments, &block)
     end
 
@@ -67,8 +65,11 @@ module Elasticsearch
           opaque_id = @opaque_id_prefix ? "#{@opaque_id_prefix}#{opaque_id}" : opaque_id
           args[4] = headers.merge('X-Opaque-Id' => opaque_id)
         end
-        verify_elasticsearch unless @verified
-        @transport.perform_request(*args, &block)
+        unless @verified
+          verify_elasticsearch(*args, &block)
+        else
+          @transport.perform_request(*args, &block)
+        end
       else
         @transport.send(name, *args, &block)
       end
@@ -80,42 +81,28 @@ module Elasticsearch
 
     private
 
-    def verify_elasticsearch
+    def verify_elasticsearch(*args, &block)
       begin
-        response = elasticsearch_validation_request
+        response = @transport.perform_request(*args, &block)
       rescue Elastic::Transport::Transport::Errors::Unauthorized,
              Elastic::Transport::Transport::Errors::Forbidden,
-             Elastic::Transport::Transport::Errors::RequestEntityTooLarge
-        @verified = true
+             Elastic::Transport::Transport::Errors::RequestEntityTooLarge => e
         warn(SECURITY_PRIVILEGES_VALIDATION_WARNING)
-        return
-      rescue Elastic::Transport::Transport::Error
+        @verified = true
+        raise e
+      rescue Elastic::Transport::Transport::Error => e
+        unless @warned
+          warn(VALIDATION_WARNING)
+          @warned = true
+        end
+        raise e
+      rescue StandardError => e
         warn(VALIDATION_WARNING)
-        return
+        raise e
       end
-
-      body = if response.headers['content-type'] == 'application/yaml'
-               require 'yaml'
-               YAML.safe_load(response.body)
-             else
-               response.body
-             end
-      version = body.dig('version', 'number')
-      verify_with_version_or_header(version, response.headers)
-    rescue StandardError => e
-      warn(VALIDATION_WARNING)
-      raise e
-    end
-
-    def verify_with_version_or_header(version, headers)
-      if version.nil? ||
-         Gem::Version.new(version) < Gem::Version.new('8.0.0.pre') && version != '8.0.0-SNAPSHOT' ||
-         headers['x-elastic-product'] != 'Elasticsearch'
-
-        raise Elasticsearch::UnsupportedProductError
-      end
-
+      raise Elasticsearch::UnsupportedProductError unless response.headers['x-elastic-product'] == 'Elasticsearch'
       @verified = true
+      response
     end
 
     def setup_cloud_host(cloud_id, user, password, port)
@@ -149,6 +136,15 @@ module Elasticsearch
       end
     end
 
+    def setup_cloud(arguments)
+      arguments[:hosts] = setup_cloud_host(
+        arguments[:cloud_id],
+        arguments[:user],
+        arguments[:password],
+        arguments[:port]
+      )
+    end
+
     # Encode credentials for the Authorization Header
     # Credentials is the base64 encoding of id and api_key joined by a colon
     # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html
@@ -159,6 +155,25 @@ module Elasticsearch
     def elasticsearch_validation_request
       @transport.perform_request('GET', '/')
     end
+
+    def sent_user_agent?(arguments)
+      return unless (headers = arguments&.[](:transport_options)&.[](:headers))
+      !!headers.keys.detect { |h| h =~ /user-?_?agent/ }
+    end
+
+    def set_user_agent!(arguments)
+      user_agent = [
+        "elasticsearch-ruby/#{Elasticsearch::VERSION}",
+        "elastic-transport-ruby/#{Elastic::Transport::VERSION}",
+        "RUBY_VERSION: #{RUBY_VERSION}"
+      ]
+      if RbConfig::CONFIG && RbConfig::CONFIG['host_os']
+        user_agent << "#{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+      end
+      arguments[:transport_options] ||= {}
+      arguments[:transport_options][:headers] ||= {}
+      arguments[:transport_options][:headers].merge!({ user_agent: user_agent.join('; ')})
+    end
   end
 
   # Error class for when we detect an unsupported version of Elasticsearch

data/spec/integration/helpers/bulk_helper_spec.rb

@@ -0,0 +1,211 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ELASTICSEARCH_URL = ENV['TEST_ES_SERVER'] || "http://localhost:#{(ENV['PORT'] || 9200)}"
+raise URI::InvalidURIError unless ELASTICSEARCH_URL =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
+
+require 'elasticsearch/helpers/bulk_helper'
+require 'spec_helper'
+require 'tempfile'
+
+context 'Elasticsearch client helpers' do
+  context 'Bulk helper' do
+    let(:client) do
+      Elasticsearch::Client.new(
+        host: ELASTICSEARCH_URL,
+        user: 'elastic',
+        password: 'changeme'
+      )
+    end
+    let(:index) { 'bulk_animals' }
+    let(:params) { { refresh: 'wait_for' } }
+    let(:bulk_helper) { Elasticsearch::Helpers::BulkHelper.new(client, index, params) }
+    let(:docs) do
+      [
+        { scientific_name: 'Lama guanicoe', name:'Guanaco' },
+        { scientific_name: 'Tayassu pecari', name:'White-lipped peccary' },
+        { scientific_name: 'Snycerus caffer', name:'Buffalo, african' },
+        { scientific_name: 'Coluber constrictor', name:'Snake, racer' },
+        { scientific_name: 'Thalasseus maximus', name:'Royal tern' },
+        { scientific_name: 'Centrocercus urophasianus', name:'Hen, sage' },
+        { scientific_name: 'Sitta canadensis', name:'Nuthatch, red-breasted' },
+        { scientific_name: 'Aegypius tracheliotus', name:'Vulture, lappet-faced' },
+        { scientific_name: 'Bucephala clangula', name:'Common goldeneye' },
+        { scientific_name: 'Felis pardalis', name:'Ocelot' }
+      ]
+    end
+
+    after do
+      client.indices.delete(index: index, ignore: 404)
+    end
+
+    it 'Ingests documents' do
+      response = bulk_helper.ingest(docs)
+      expect(response).to be_an_instance_of Elasticsearch::API::Response
+      expect(response.status).to eq(200)
+      expect(response['items'].map { |a| a['index']['status'] }.uniq.first).to eq 201
+    end
+
+    it 'Updates documents' do
+      docs = [
+        { scientific_name: 'Otocyon megalotos', name: 'Bat-eared fox' },
+        { scientific_name: 'Herpestes javanicus', name: 'Small Indian mongoose' }
+      ]
+      bulk_helper.ingest(docs)
+      # Get the ingested documents, add id and modify them to update them:
+      animals = client.search(index: index)['hits']['hits']
+      # Add id to each doc
+      docs = animals.map { |animal| animal['_source'].merge({'id' => animal['_id'] }) }
+      docs.map { |doc| doc['scientific_name'].upcase! }
+      response = bulk_helper.update(docs)
+      expect(response.status).to eq(200)
+      expect(response['items'].map { |i| i['update']['result'] }.uniq.first).to eq('updated')
+    end
+
+    it 'Deletes documents' do
+      response = bulk_helper.ingest(docs)
+      ids = response.body['items'].map { |a| a['index']['_id'] }
+      response = bulk_helper.delete(ids)
+      expect(response.status).to eq 200
+      expect(response['items'].map { |item| item['delete']['result'] }.uniq.first).to eq('deleted')
+      expect(client.count(index: index)['count']).to eq(0)
+    end
+
+    it 'Ingests documents and yields response and docs' do
+      slice = 2
+      response = bulk_helper.ingest(docs, {slice: slice}) do |response, docs|
+        expect(response).to be_an_instance_of Elasticsearch::API::Response
+        expect(docs.count).to eq slice
+      end
+    end
+
+    context 'JSON File helper' do
+      let(:file) { Tempfile.new('test-data.json') }
+      let(:json) do
+        json = <<~JSON
+        [
+          {
+            "character_name": "Anallese Lonie",
+           "species": "mouse",
+           "catchphrase": "Seamless regional definition",
+           "favorite_food": "pizza"
+          },
+          {
+            "character_name": "Janey Davidovsky",
+           "species": "cat",
+           "catchphrase": "Down-sized responsive pricing structure",
+           "favorite_food": "pizza"
+          },
+          {
+            "character_name": "Morse Mountford",
+           "species": "cat",
+           "catchphrase": "Ameliorated modular data-warehouse",
+           "favorite_food": "carrots"
+          },
+          {
+            "character_name": "Saundra Kauble",
+           "species": "dog",
+           "catchphrase": "Synchronised 24/7 support",
+           "favorite_food": "carrots"
+          },
+          {
+            "character_name": "Kain Viggars",
+           "species": "cat",
+           "catchphrase": "Open-architected asymmetric circuit",
+           "favorite_food": "carrots"
+          }
+        ]
+        JSON
+      end
+
+      before do
+        file.write(json)
+        file.rewind
+      end
+
+      after do
+        file.close
+        file.unlink
+      end
+
+      it 'Ingests a JSON file' do
+        response = bulk_helper.ingest_json(file)
+
+        expect(response).to be_an_instance_of Elasticsearch::API::Response
+        expect(response.status).to eq(200)
+      end
+
+      context 'with data not in root of JSON file' do
+        let(:json) do
+          json = <<~JSON
+          {
+            "field": "value",
+            "status": 200,
+            "data": {
+              "items": [
+                {
+                  "character_name": "Anallese Lonie",
+                  "species": "mouse",
+                  "catchphrase": "Seamless regional definition",
+                  "favorite_food": "pizza"
+                },
+                {
+                  "character_name": "Janey Davidovsky",
+                  "species": "cat",
+                  "catchphrase": "Down-sized responsive pricing structure",
+                  "favorite_food": "pizza"
+                },
+                {
+                  "character_name": "Morse Mountford",
+                  "species": "cat",
+                  "catchphrase": "Ameliorated modular data-warehouse",
+                  "favorite_food": "carrots"
+                },
+                {
+                  "character_name": "Saundra Kauble",
+                  "species": "dog",
+                  "catchphrase": "Synchronised 24/7 support",
+                  "favorite_food": "carrots"
+                },
+                {
+                  "character_name": "Kain Viggars",
+                  "species": "cat",
+                  "catchphrase": "Open-architected asymmetric circuit",
+                  "favorite_food": "carrots"
+                }
+              ]
+            }
+          }
+          JSON
+        end
+
+        it 'Ingests a JSON file passing keys as Array' do
+          response = bulk_helper.ingest_json(file, { keys: ['data', 'items'] })
+          expect(response).to be_an_instance_of Elasticsearch::API::Response
+          expect(response.status).to eq(200)
+          expect(response['items'].map { |a| a['index']['status'] }.uniq.first).to eq 201
+        end
+
+        it 'Ingests a JSON file passing keys as String' do
+          response = bulk_helper.ingest_json(file, { keys: 'data,items' })
+          expect(response).to be_an_instance_of Elasticsearch::API::Response
+          expect(response.status).to eq(200)
+          expect(response['items'].map { |a| a['index']['status'] }.uniq.first).to eq 201
+        end
+      end
+    end
+  end
+end

data/spec/integration/helpers/scroll_helper_spec.rb

@@ -0,0 +1,91 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ELASTICSEARCH_URL = ENV['TEST_ES_SERVER'] || "http://localhost:#{(ENV['PORT'] || 9200)}"
+raise URI::InvalidURIError unless ELASTICSEARCH_URL =~ /\A#{URI::DEFAULT_PARSER.make_regexp}\z/
+
+require 'spec_helper'
+require 'elasticsearch/helpers/scroll_helper'
+
+context 'Elasticsearch client helpers' do
+  let(:client) do
+    Elasticsearch::Client.new(
+      host: ELASTICSEARCH_URL,
+      user: 'elastic',
+      password: 'changeme'
+    )
+  end
+  let(:index) { 'books' }
+  let(:body) { { size: 12, query: { match_all: {} } } }
+  let(:scroll_helper) { Elasticsearch::Helpers::ScrollHelper.new(client, index, body) }
+
+  before do
+    documents = [
+      { index: { _index: index, data: {name: "Leviathan Wakes", "author": "James S.A. Corey", "release_date": "2011-06-02", "page_count": 561} } },
+      { index: { _index: index, data: {name: "Hyperion", "author": "Dan Simmons", "release_date": "1989-05-26", "page_count": 482} } },
+      { index: { _index: index, data: {name: "Dune", "author": "Frank Herbert", "release_date": "1965-06-01", "page_count": 604} } },
+      { index: { _index: index, data: {name: "Dune Messiah", "author": "Frank Herbert", "release_date": "1969-10-15", "page_count": 331} } },
+      { index: { _index: index, data: {name: "Children of Dune", "author": "Frank Herbert", "release_date": "1976-04-21", "page_count": 408} } },
+      { index: { _index: index, data: {name: "God Emperor of Dune", "author": "Frank Herbert", "release_date": "1981-05-28", "page_count": 454} } },
+      { index: { _index: index, data: {name: "Consider Phlebas", "author": "Iain M. Banks", "release_date": "1987-04-23", "page_count": 471} } },
+      { index: { _index: index, data: {name: "Pandora's Star", "author": "Peter F. Hamilton", "release_date": "2004-03-02", "page_count": 768} } },
+      { index: { _index: index, data: {name: "Revelation Space", "author": "Alastair Reynolds", "release_date": "2000-03-15", "page_count": 585} } },
+      { index: { _index: index, data: {name: "A Fire Upon the Deep", "author": "Vernor Vinge", "release_date": "1992-06-01", "page_count": 613} } },
+      { index: { _index: index, data: {name: "Ender's Game", "author": "Orson Scott Card", "release_date": "1985-06-01", "page_count": 324} } },
+      { index: { _index: index, data: {name: "1984", "author": "George Orwell", "release_date": "1985-06-01", "page_count": 328} } },
+      { index: { _index: index, data: {name: "Fahrenheit 451", "author": "Ray Bradbury", "release_date": "1953-10-15", "page_count": 227} } },
+      { index: { _index: index, data: {name: "Brave New World", "author": "Aldous Huxley", "release_date": "1932-06-01", "page_count": 268} } },
+      { index: { _index: index, data: {name: "Foundation", "author": "Isaac Asimov", "release_date": "1951-06-01", "page_count": 224} } },
+      { index: { _index: index, data: {name: "The Giver", "author": "Lois Lowry", "release_date": "1993-04-26", "page_count": 208} } },
+      { index: { _index: index, data: {name: "Slaughterhouse-Five", "author": "Kurt Vonnegut", "release_date": "1969-06-01", "page_count": 275} } },
+      { index: { _index: index, data: {name: "The Hitchhiker's Guide to the Galaxy", "author": "Douglas Adams", "release_date": "1979-10-12", "page_count": 180} } },
+      { index: { _index: index, data: {name: "Snow Crash", "author": "Neal Stephenson", "release_date": "1992-06-01", "page_count": 470} } },
+      { index: { _index: index, data: {name: "Neuromancer", "author": "William Gibson", "release_date": "1984-07-01", "page_count": 271} } },
+      { index: { _index: index, data: {name: "The Handmaid's Tale", "author": "Margaret Atwood", "release_date": "1985-06-01", "page_count": 311} } },
+      { index: { _index: index, data: {name: "Starship Troopers", "author": "Robert A. Heinlein", "release_date": "1959-12-01", "page_count": 335} } },
+      { index: { _index: index, data: {name: "The Left Hand of Darkness", "author": "Ursula K. Le Guin", "release_date": "1969-06-01", "page_count": 304} } },
+      { index: { _index: index, data: {name: "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288 } } }
+    ]
+    client.bulk(body: documents, refresh: 'wait_for')
+  end
+
+  after do
+    client.indices.delete(index: index)
+  end
+
+  it 'instantiates a scroll helper' do
+    expect(scroll_helper).to be_an_instance_of Elasticsearch::Helpers::ScrollHelper
+  end
+
+  it 'searches an index' do
+    my_documents = []
+    while !(documents = scroll_helper.results).empty?
+      my_documents << documents
+    end
+
+    expect(my_documents.flatten.size).to eq 24
+  end
+
+  it 'uses enumerable' do
+    count = 0
+    scroll_helper.each { |a| count += 1 }
+    expect(count).to eq 24
+    expect(scroll_helper).to respond_to(:count)
+    expect(scroll_helper).to respond_to(:reject)
+    expect(scroll_helper).to respond_to(:uniq)
+    expect(scroll_helper.map { |a| a['_id'] }.uniq.count).to eq 24
+  end
+end

data/spec/unit/elasticsearch_product_validation_spec.rb

@@ -20,52 +20,28 @@ require 'webmock/rspec'
 
 describe 'Elasticsearch: Validation' do
   let(:host) { 'http://localhost:9200' }
-  let(:verify_request_stub) do
-    stub_request(:get, host)
-      .to_return(status: status, body: body, headers: headers)
-  end
   let(:count_request_stub) do
     stub_request(:get, "#{host}/_count")
-      .to_return(status: 200, body: nil, headers: {})
+      .to_return(status: status, body: nil, headers: headers)
   end
   let(:status) { 200 }
-  let(:body) { {}.to_json }
+  let(:body) { nil }
+  let(:headers) { {} }
   let(:client) { Elasticsearch::Client.new }
-  let(:headers) do
-    { 'content-type' => 'application/json' }
-  end
-
-  def error_requests_and_expectations(message = Elasticsearch::NOT_ELASTICSEARCH_WARNING)
-    expect { client.count }.to raise_error Elasticsearch::UnsupportedProductError, message
-    assert_requested :get, host
-    assert_not_requested :post, "#{host}/_count"
-    expect { client.cluster.health }.to raise_error Elasticsearch::UnsupportedProductError, message
-    expect(client.instance_variable_get('@verified')).to be false
-    expect { client.cluster.health }.to raise_error Elasticsearch::UnsupportedProductError, message
-  end
-
-  def valid_requests_and_expectations
-    expect(client.instance_variable_get('@verified')).to be false
-    assert_not_requested :get, host
-
-    client.count
-    expect(client.instance_variable_get('@verified'))
-    assert_requested :get, host
-    assert_requested :get, "#{host}/_count"
-  end
 
   context 'When Elasticsearch replies with status 401' do
     let(:status) { 401 }
-    let(:body) { {}.to_json }
 
     it 'Verifies the request but shows a warning' do
       stderr      = $stderr
       fake_stderr = StringIO.new
       $stderr     = fake_stderr
-
-      verify_request_stub
+      expect(client.instance_variable_get('@verified')).to be false
       count_request_stub
-      valid_requests_and_expectations
+      expect do
+        client.count
+      end.to raise_error Elastic::Transport::Transport::Errors::Unauthorized
+      expect(client.instance_variable_get('@verified')).to be true
 
       fake_stderr.rewind
       expect(fake_stderr.string).to eq("#{Elasticsearch::SECURITY_PRIVILEGES_VALIDATION_WARNING}\n")
@@ -76,16 +52,18 @@ describe 'Elasticsearch: Validation' do
 
   context 'When Elasticsearch replies with status 403' do
     let(:status) { 403 }
-    let(:body) { {}.to_json }
 
     it 'Verifies the request but shows a warning' do
       stderr      = $stderr
       fake_stderr = StringIO.new
       $stderr     = fake_stderr
 
-      verify_request_stub
+      expect(client.instance_variable_get('@verified')).to be false
       count_request_stub
-      valid_requests_and_expectations
+      expect do
+        client.count
+      end.to raise_error Elastic::Transport::Transport::Errors::Forbidden
+      expect(client.instance_variable_get('@verified')).to be true
 
       fake_stderr.rewind
       expect(fake_stderr.string).to eq("#{Elasticsearch::SECURITY_PRIVILEGES_VALIDATION_WARNING}\n")
@@ -96,7 +74,6 @@ describe 'Elasticsearch: Validation' do
 
   context 'When Elasticsearch replies with status 413' do
     let(:status) { 413 }
-    let(:body) { {}.to_json }
 
     it 'Verifies the request and shows a warning' do
       stderr      = $stderr
@@ -104,9 +81,10 @@ describe 'Elasticsearch: Validation' do
       $stderr     = fake_stderr
 
       expect(client.instance_variable_get('@verified')).to be false
-      assert_not_requested :get, host
-      verify_request_stub
-      expect { client.info }.to raise_error Elastic::Transport::Transport::Errors::RequestEntityTooLarge
+      count_request_stub
+      expect do
+        client.count
+      end.to raise_error Elastic::Transport::Transport::Errors::RequestEntityTooLarge
       expect(client.instance_variable_get('@verified')).to be true
 
       fake_stderr.rewind
@@ -127,10 +105,10 @@ describe 'Elasticsearch: Validation' do
       $stderr     = fake_stderr
 
       expect(client.instance_variable_get('@verified')).to be false
-      assert_not_requested :get, host
-      verify_request_stub
-      expect { client.info }.to raise_error Elastic::Transport::Transport::Errors::ServiceUnavailable
-      assert_not_requested :post, "#{host}/_count"
+      count_request_stub
+      expect do
+        client.count
+      end.to raise_error Elastic::Transport::Transport::Errors::ServiceUnavailable
       expect(client.instance_variable_get('@verified')).to be false
 
       fake_stderr.rewind
@@ -147,156 +125,24 @@ describe 'Elasticsearch: Validation' do
     end
   end
 
+  context 'When the header is present' do
+    let(:headers) { { 'X-Elastic-Product' => 'Elasticsearch' } }
 
-  context 'When the Elasticsearch version is >= 8.0.0' do
-    context 'With a valid Elasticsearch response' do
-      let(:body) { { 'version' => { 'number' => '8.0.0' } }.to_json }
-      let(:headers) do
-        {
-          'X-Elastic-Product' => 'Elasticsearch',
-          'content-type' => 'json'
-        }
-      end
-
-      it 'Makes requests and passes validation' do
-        verify_request_stub
-        count_request_stub
-
-        valid_requests_and_expectations
-      end
-    end
-
-    context 'When the header is not present' do
-      it 'Fails validation' do
-        verify_request_stub
-
-        expect(client.instance_variable_get('@verified')).to be false
-        assert_not_requested :get, host
-
-        error_requests_and_expectations
-      end
-    end
-  end
-
-  context 'When the Elasticsearch version is >= 8.1.0' do
-    context 'With a valid Elasticsearch response' do
-      let(:body) { { 'version' => { 'number' => '8.1.0' } }.to_json }
-      let(:headers) do
-        {
-          'X-Elastic-Product' => 'Elasticsearch',
-          'content-type' => 'json'
-        }
-      end
-
-      it 'Makes requests and passes validation' do
-        verify_request_stub
-        count_request_stub
-
-        valid_requests_and_expectations
-      end
-    end
-
-    context 'When the header is not present' do
-      it 'Fails validation' do
-        verify_request_stub
-
-        expect(client.instance_variable_get('@verified')).to be false
-        assert_not_requested :get, host
-
-        error_requests_and_expectations
-      end
-    end
-  end
-
-
-  context 'When the Elasticsearch version is 8.0.0.pre' do
-    context 'With a valid Elasticsearch response' do
-      let(:body) { { 'version' => { 'number' => '8.0.0.pre' } }.to_json }
-      let(:headers) do
-        {
-          'X-Elastic-Product' => 'Elasticsearch',
-          'content-type' => 'json'
-        }
-      end
-
-      it 'Makes requests and passes validation' do
-        verify_request_stub
-        count_request_stub
-
-        valid_requests_and_expectations
-      end
-    end
-
-    context 'When the header is not present' do
-      it 'Fails validation' do
-        verify_request_stub
-
-        expect(client.instance_variable_get('@verified')).to be false
-        assert_not_requested :get, host
-
-        error_requests_and_expectations
-      end
-    end
-  end
-
-  context 'When the version is 8.0.0-SNAPSHOT' do
-    let(:body) { { 'version' => { 'number' => '8.0.0-SNAPSHOT' } }.to_json }
-
-    context 'When the header is not present' do
-      it 'Fails validation' do
-        verify_request_stub
-        count_request_stub
-
-        error_requests_and_expectations
-      end
-    end
-
-    context 'With a valid Elasticsearch response' do
-      let(:headers) do
-        {
-          'X-Elastic-Product' => 'Elasticsearch',
-          'content-type' => 'json'
-        }
-      end
-
-      it 'Makes requests and passes validation' do
-        verify_request_stub
-        count_request_stub
-
-        valid_requests_and_expectations
-      end
-    end
-  end
-
-  context 'When Elasticsearch version is < 8.0.0' do
-    let(:body) { { 'version' => { 'number' => '7.16.0' } }.to_json }
-
-    it 'Raises an exception and client doesnae work' do
-      verify_request_stub
-      error_requests_and_expectations
-    end
-  end
-
-  context 'When there is no version data' do
-    let(:body) { {}.to_json }
-    it 'Raises an exception and client doesnae work' do
-      verify_request_stub
-      error_requests_and_expectations
+    it 'Makes requests and passes validation' do
+      expect(client.instance_variable_get('@verified')).to be false
+      count_request_stub
+      client.count
+      expect(client.instance_variable_get('@verified')).to be true
     end
   end
 
-  context 'When doing a yaml content-type request' do
-    let(:client) do
-      Elasticsearch::Client.new(transport_options: {headers: { accept: 'application/yaml', content_type: 'application/yaml' }})
-    end
-
-    let(:headers) { { 'content-type' => 'application/yaml', 'X-Elastic-Product' => 'Elasticsearch' } }
-    let(:body) { "---\nversion:\n  number: \"8.0.0-SNAPSHOT\"\n" }
-
-    it 'validates' do
-      verify_request_stub
-      count_request_stub
-      valid_requests_and_expectations
+  context 'When the header is not present' do
+    it 'Fails validation' do
+      expect(client.instance_variable_get('@verified')).to be false
+      stub_request(:get, "#{host}/_cluster/health")
+        .to_return(status: status, body: nil, headers: {})
+      expect { client.cluster.health }.to raise_error Elasticsearch::UnsupportedProductError, Elasticsearch::NOT_ELASTICSEARCH_WARNING
+      expect(client.instance_variable_get('@verified')).to be false
     end
   end
 end

data/spec/unit/user_agent_spec.rb

@@ -0,0 +1,69 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+require 'spec_helper'
+
+describe Elasticsearch::Client do
+  let(:user_agent) {
+    "elasticsearch-ruby/#{Elasticsearch::VERSION}; elastic-transport-ruby/#{Elastic::Transport::VERSION}; RUBY_VERSION: #{RUBY_VERSION}; #{RbConfig::CONFIG['host_os'].split('_').first[/[a-z]+/i].downcase} #{RbConfig::CONFIG['target_cpu']}"
+  }
+
+  context 'when no user-agent is set on initialization' do
+    let(:client) { described_class.new }
+
+    it 'has the expected header' do
+      expect(client.transport.options[:transport_options][:headers][:user_agent]).to eq user_agent
+    end
+  end
+
+  context 'when a header is specified on initialization' do
+    let(:client) do
+      described_class.new(
+        transport_options: { headers: { 'X-Test-Header' => 'Test' } }
+      )
+    end
+
+    it 'has the expected header' do
+      expect(client.transport.options[:transport_options][:headers][:user_agent]).to eq user_agent
+      expect(client.transport.options[:transport_options][:headers]['X-Test-Header']).to eq 'Test'
+    end
+  end
+
+  context 'when other transport_options are specified on initialization' do
+    let(:client) do
+      described_class.new(
+        transport_options: { params:  { format: 'yaml' } }
+      )
+    end
+
+    it 'has the expected header' do
+      expect(client.transport.options[:transport_options][:headers][:user_agent]).to eq user_agent
+      expect(client.transport.options[:transport_options][:params][:format]).to eq 'yaml'
+    end
+  end
+
+  context 'when :user_agent is specified on initialization' do
+    let(:client) do
+      described_class.new(
+        transport_options: { headers: { user_agent: 'TestApp' } }
+      )
+    end
+
+    it 'has the expected header' do
+      expect(client.transport.options[:transport_options][:headers][:user_agent]).to eq 'TestApp'
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch
 version: !ruby/object:Gem::Version
-  version: 8.8.0
+  version: 8.9.0
 platform: ruby
 authors:
 - Karel Minarik
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-05-25 00:00:00.000000000 Z
+date: 2023-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elastic-transport
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 8.8.0
+        version: 8.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 8.8.0
+        version: 8.9.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -199,9 +199,13 @@ files:
 - elasticsearch.gemspec
 - lib/elasticsearch-ruby.rb
 - lib/elasticsearch.rb
+- lib/elasticsearch/helpers/bulk_helper.rb
+- lib/elasticsearch/helpers/scroll_helper.rb
 - lib/elasticsearch/version.rb
 - spec/integration/characters_escaping_spec.rb
 - spec/integration/client_integration_spec.rb
+- spec/integration/helpers/bulk_helper_spec.rb
+- spec/integration/helpers/scroll_helper_spec.rb
 - spec/spec_helper.rb
 - spec/unit/api_key_spec.rb
 - spec/unit/cloud_credentials_spec.rb
@@ -209,6 +213,7 @@ files:
 - spec/unit/elasticsearch_product_validation_spec.rb
 - spec/unit/headers_spec.rb
 - spec/unit/opaque_id_spec.rb
+- spec/unit/user_agent_spec.rb
 - spec/unit/wrapper_gem_spec.rb
 homepage: https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html
 licenses:
@@ -241,6 +246,8 @@ summary: Ruby integrations for Elasticsearch
 test_files:
 - spec/integration/characters_escaping_spec.rb
 - spec/integration/client_integration_spec.rb
+- spec/integration/helpers/bulk_helper_spec.rb
+- spec/integration/helpers/scroll_helper_spec.rb
 - spec/spec_helper.rb
 - spec/unit/api_key_spec.rb
 - spec/unit/cloud_credentials_spec.rb
@@ -248,4 +255,5 @@ test_files:
 - spec/unit/elasticsearch_product_validation_spec.rb
 - spec/unit/headers_spec.rb
 - spec/unit/opaque_id_spec.rb
+- spec/unit/user_agent_spec.rb
 - spec/unit/wrapper_gem_spec.rb