fluent-plugin-elasticsearch 4.1.1 → 4.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 616365bcf32cc6d302718907d145639cbb4786b33cb8a837da24f3934cf7a837
-  data.tar.gz: 77df51b64aa07d12b57b82a093f4a5b8cbd1c1bf9c5c82384163118c050c68cf
+  metadata.gz: 9eb6b06e90f827162d669e94fb9a622c09f12b35e0a05f82c961ad870e3f9726
+  data.tar.gz: c7964330b33c36176e4b1238e4ba4f2d55e60e6ee6da4fec6799f25d943ab85f
 SHA512:
-  metadata.gz: ae68cc2f0a2abbf7cf7a4c11a9bb5dac9a9e09ba9c053109748c671acd304198b45ad949782691a28eadb418e15bc66f1ef16897f810c5f0b5dd0ce04dd7913e
-  data.tar.gz: 484df0670840045d6c55bd1971142cf13a2a0c35cefc0ff96ada3259404d08529d97adbdd736bf9eaf99ad84c51d2124af1d0160c35042a736c875eef8aa5f3c
+  metadata.gz: fd5d961731f5866a7c841f9cbeb11f8434868988862cb9929af4cfbe174a7ce65c38d125b91dc8952120be2da4b5ad08e89c127f9c65b9e38585185acccd1cdc
+  data.tar.gz: 0c15c5670b0a2a9eda90b6962768c8db7e510e2da4b2434e3cb841d2fa5b15cf52a8f65f35b4236ab01cb8721d640c690f3bf753a4f4f269b2c802d9f2d7a9b6

data/.github/workflows/linux.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6' ]
+        ruby: [ '2.4', '2.5', '2.6', '2.7' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,6 +2,27 @@
 
 ### [Unreleased]
 
+### 4.2.1
+- Update a broken link (#821)
+- Include chunk_id in records (#820)
+- Fix a failing testcase (#818)
+
+### 4.2.0
+- ci: Add Ruby 2.7 jobs (#812)
+- Support Elasticsearch new style template (#810)
+
+### 4.1.4
+- ElasticsearchGenID update docs for hash_type (#809)
+- Handle api key header (#808)
+- index_template: Handle error object entirely on index creation failure (#807)
+
+### 4.1.3
+- Load multiple templates even if template_name and template_file given (#799)
+- Handle elasticsearch-ruby 7.9.0 using HTTP method changes (#795)
+
+### 4.1.2
+- Use Hash#dig instead of Hash#[] on retrieving version information from Elasticsearch info API (#793)
+
 ### 4.1.1
 - Correct ILM explain on logstash_format case (#786)
 

data/README.ElasticsearchGenID.md

@@ -26,7 +26,7 @@ In your Fluentd configuration, use `@type elasticsearch_genid`. Additional confi
   use_entire_record false
   record_keys []
   separator _
-  hash_type md5
+  hash_type sha1
 </match>
 ```
 
@@ -95,7 +95,7 @@ You can specify separator charactor to creating seed for hash generation.
 hash_type sha1
 ```
 
-You can specify hash algorithm.
+You can specify hash algorithm. Support algorithms `md5`, `sha1`, `sha256`, `sha512`. Default: `sha1`
 
 ## Advanced Usage
 

data/README.md

@@ -85,6 +85,7 @@ Current maintainers: @cosmo0920
   + [verify_es version at startup](#verify_es_version_at_startup)
   + [default_elasticsearch_version](#default_elasticsearch_version)
   + [custom_headers](#custom_headers)
+  + [api_key](#api_key)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
   + [Placeholders](#placeholders)
@@ -100,6 +101,10 @@ Current maintainers: @cosmo0920
   + [ilm_policies](#ilm_policies)
   + [ilm_policy_overwrite](#ilm_policy_overwrite)
   + [truncate_caches_interval](#truncate_caches_interval)
+  + [use_legacy_template](#use_legacy_template)
+  + [metadata section](#metadata-section)
+    + [include_chunk_id](#include_chunk_id)
+    + [chunk_id_key](#chunk_id_key)
 * [Configuration - Elasticsearch Input](#configuration---elasticsearch-input)
 * [Configuration - Elasticsearch Filter GenID](#configuration---elasticsearch-filter-genid)
 * [Elasticsearch permissions](#elasticsearch-permissions)
@@ -121,10 +126,12 @@ Current maintainers: @cosmo0920
 
 ## Requirements
 
-| fluent-plugin-elasticsearch  | fluentd | ruby |
-|-------------------|---------|------|
-| >= 2.0.0 | >= v0.14.20 | >= 2.1 |
-|  < 2.0.0 | >= v0.12.0 | >= 1.9 |
+| fluent-plugin-elasticsearch  | fluentd     | ruby   |
+|:----------------------------:|:-----------:|:------:|
+| >= 4.0.1                     | >= v0.14.22 | >= 2.3 |
+| >= 3.2.4 && < 4.0.1          | >= v0.14.22 | >= 2.1 |
+| >= 2.0.0 && < 3.2.3          | >= v0.14.20 | >= 2.1 |
+|  < 2.0.0                     | >= v0.12.0  | >= 1.9 |
 
 NOTE: For v0.12 version, you should use 1.x.y version. Please send patch into v0.12 branch if you encountered 1.x version's bug.
 
@@ -429,7 +436,7 @@ Specify index templates in form of hash. Can contain multiple templates.
 templates { "template_name_1": "path_to_template_1_file", "template_name_2": "path_to_template_2_file"}
 ```
 
-If `template_file` and `template_name` are set, then this parameter will be ignored.
+**Note:** Before ES plugin v4.1.2, if `template_file` and `template_name` are set, then this parameter will be ignored. In 4.1.3 or later, `template_file` and `template_name` can work with `templates`.
 
 ### customize_template
 
@@ -917,7 +924,7 @@ Starting with version 0.8.0, this gem uses excon, which supports proxy with envi
 
 ### Buffer options
 
-`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/v0.14/articles/output-plugin-overview) and use `compat_parameters` plugin helper. It adds the following options:
+`fluentd-plugin-elasticsearch` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/output#overview) and use `compat_parameters` plugin helper. It adds the following options:
 
 ```
 buffer_type memory
@@ -1104,6 +1111,14 @@ This parameter adds additional headers to request. The default value is `{}`.
 custom_headers {"token":"secret"}
 ```
 
+### api_key
+
+This parameter adds authentication header. The default value is `nil`.
+
+```
+api_key "ElasticsearchAPIKEY"
+```
+
 ### Not seeing a config you need?
 
 We try to keep the scope of this plugin small and not add too many configuration options. If you think an option would be useful to others, feel free to open an issue or contribute a Pull Request.
@@ -1292,6 +1307,63 @@ If it is set, timer for clearing `alias_indexes` and `template_names` caches wil
 
 Default value is `nil`.
 
+## use_legacy_template
+
+Use legacy template or not.
+
+Elasticsearch 7.8 or later supports the brand new composable templates.
+
+For Elasticsearch 7.7 or older, users should specify this parameter as `false`.
+
+Composable template documentation is [Put Index Template API | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html) and legacy template documentation is [Index Templates | Elasticsearch Reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates-v1.html).
+
+Please confirm that whether the using Elasticsearch cluster(s) support the composable template feature or not when turn on the brand new feature with this parameter.
+
+## <metadata\> section
+
+Users can specify whether including `chunk_id` information into records or not:
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id true
+    # chunk_id_key chunk_id # Default value is "chunk_id".
+  </metadata>
+</match>
+```
+
+### include_chunk_id
+
+Whether including `chunk_id` for not. Default value is `false`.
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id true
+  </metadata>
+</match>
+```
+
+
+### chunk_id_key
+
+Specify `chunk_id_key` to store `chunk_id` information into records. Default value is `chunk_id`.
+
+```aconf
+<match your.awesome.routing.tag>
+  @type elasticsearch
+  # Other configurations.
+  <metadata>
+    include_chunk_id
+    chunk_id_key chunk_hex
+  </metadata>
+</match>
+```
+
 ## Configuration - Elasticsearch Input
 
 See [Elasticsearch Input plugin document](README.ElasticsearchInput.md)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.1.1'
+  s.version       = '4.2.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -22,7 +22,11 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_exists?(name, host = nil)
-    client(host).indices.get_template(:name => name)
+    if @use_legacy_template
+      client(host).indices.get_template(:name => name)
+    else
+      client(host).indices.get_index_template(:name => name)
+    end
     return true
   rescue Elasticsearch::Transport::Transport::Errors::NotFound
     return false
@@ -52,7 +56,11 @@ module Fluent::ElasticsearchIndexTemplate
   end
 
   def template_put(name, template, host = nil)
-    client(host).indices.put_template(:name => name, :body => template)
+    if @use_legacy_template
+      client(host).indices.put_template(:name => name, :body => template)
+    else
+      client(host).indices.put_index_template(:name => name, :body => template)
+    end
   end
 
   def indexcreation(index_name, host = nil)
@@ -61,7 +69,7 @@ module Fluent::ElasticsearchIndexTemplate
     if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else
-      log.error("Error while index creation - #{index_name}: #{e.inspect}")
+      log.error("Error while index creation - #{index_name}", error: e)
     end
   end
 
@@ -121,13 +129,29 @@ module Fluent::ElasticsearchIndexTemplate
 
   def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
     log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
-    template.delete('template') if template.include?('template')
     template['index_patterns'] = "#{target_index}-*"
-    template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
-    if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
-      log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+    if @use_legacy_template
+      template.delete('template') if template.include?('template')
+      # Prepare settings Hash
+      if !template.key?('settings')
+        template['settings'] = {}
+      end
+      if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['order'] = template['order'] ? template['order'] + target_index.split('-').length : 50 + target_index.split('-').length
+    else
+      # Prepare template.settings Hash
+      if !template['template'].key?('settings')
+        template['template']['settings'] = {}
+      end
+      if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
+        log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
+      end
+      template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
+      template['priority'] = template['priority'] ? template['priority'] + target_index.split('-').length : 100 + target_index.split('-').length
     end
-    template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
     template
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -8,6 +8,7 @@ rescue LoadError
 end
 require 'json'
 require 'uri'
+require 'base64'
 begin
   require 'strptime'
 rescue LoadError
@@ -17,6 +18,7 @@ require 'fluent/plugin/output'
 require 'fluent/event'
 require 'fluent/error'
 require 'fluent/time'
+require 'fluent/unique_id'
 require 'fluent/log-ext'
 require 'zlib'
 require_relative 'elasticsearch_constants'
@@ -57,6 +59,7 @@ module Fluent::Plugin
     attr_reader :template_names
     attr_reader :ssl_version_options
     attr_reader :compressable_connection
+    attr_reader :api_key_header
 
     helpers :event_emitter, :compat_parameters, :record_accessor, :timer
 
@@ -155,6 +158,7 @@ EOC
     config_param :default_elasticsearch_version, :integer, :default => DEFAULT_ELASTICSEARCH_VERSION
     config_param :log_es_400_reason, :bool, :default => false
     config_param :custom_headers, :hash, :default => {}
+    config_param :api_key, :string, :default => nil, :secret => true
     config_param :suppress_doc_wrap, :bool, :default => false
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
@@ -166,6 +170,12 @@ EOC
     config_param :ilm_policies, :hash, :default => {}
     config_param :ilm_policy_overwrite, :bool, :default => false
     config_param :truncate_caches_interval, :time, :default => nil
+    config_param :use_legacy_template, :bool, :default => true
+
+    config_section :metadata, param_name: :metainfo, multi: false do
+      config_param :include_chunk_id, :bool, :default => false
+      config_param :chunk_id_key, :string, :default => "chunk_id".freeze
+    end
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -212,6 +222,8 @@ EOC
         @remove_keys_on_update = @remove_keys_on_update.split ','
       end
 
+      @api_key_header = setup_api_key
+
       raise Fluent::ConfigError, "'max_retry_putting_template' must be greater than or equal to zero." if @max_retry_putting_template < 0
       raise Fluent::ConfigError, "'max_retry_get_es_version' must be greater than or equal to zero." if @max_retry_get_es_version < 0
 
@@ -247,7 +259,8 @@ EOC
             template_installation_actual(@deflector_alias ? @deflector_alias : @index_name, @template_name, @customize_template, @application_name, @index_name, @ilm_policy_id)
           end
           verify_ilm_working if @enable_ilm
-        elsif @templates
+        end 
+        if @templates
           retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             templates_hash_install(@templates, @template_overwrite)
           end
@@ -397,6 +410,12 @@ EOC
       end
     end
 
+    def setup_api_key
+      return {} unless @api_key
+
+      { "Authorization" => "ApiKey " + Base64.strict_encode64(@api_key) }
+    end
+
     def dry_run?
       if Fluent::Engine.respond_to?(:dry_run_mode)
         Fluent::Engine.dry_run_mode
@@ -461,7 +480,10 @@ EOC
 
     def detect_es_major_version
       @_es_info ||= client.info
-      @_es_info["version"]["number"].to_i
+      unless version = @_es_info.dig("version", "number")
+        version = @default_elasticsearch_version
+      end
+      version.to_i
     end
 
     def client_library_version
@@ -554,7 +576,10 @@ EOC
                        else
                          {}
                        end
-        headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers).merge(gzip_headers)
+        headers = { 'Content-Type' => @content_type.to_s }
+                    .merge(@custom_headers)
+                    .merge(@api_key_header)
+                    .merge(gzip_headers)
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
 
         transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
@@ -752,6 +777,15 @@ EOC
       true
     end
 
+    def inject_chunk_id_to_record_if_needed(record, chunk_id)
+      if @metainfo&.include_chunk_id
+        record[@metainfo.chunk_id_key] = chunk_id
+        record
+      else
+        record
+      end
+    end
+
     def write(chunk)
       bulk_message_count = Hash.new { |h,k| h[k] = 0 }
       bulk_message = Hash.new { |h,k| h[k] = '' }
@@ -759,6 +793,7 @@ EOC
       meta = {}
 
       tag = chunk.metadata.tag
+      chunk_id = dump_unique_id_hex(chunk.unique_id)
       extracted_values = expand_placeholders(chunk)
       host = if @hosts
                extract_placeholders(@hosts, chunk)
@@ -768,6 +803,9 @@ EOC
 
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
+
+        record = inject_chunk_id_to_record_if_needed(record, chunk_id)
+
         begin
           meta, header, record = process_message(tag, meta, header, time, record, extracted_values)
           info = if @include_index_in_url

data/test/plugin/test_elasticsearch_tls.rb

@@ -62,8 +62,14 @@ class TestElasticsearchTLS < Test::Unit::TestCase
       d = driver('')
       ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
       if @use_tls_minmax_version
-        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
-                      min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+        if @enabled_tlsv1_3
+          assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                        min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+        else
+          assert_equal({max_version: nil,
+                        min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+
+        end
       else
         assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
       end

data/test/plugin/test_in_elasticsearch.rb

@@ -20,6 +20,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
+    @http_method = if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("7.9.0")
+                     :post
+                   else
+                     :get
+                   end
   end
 
   def driver(conf='')
@@ -313,7 +318,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
   end
 
   def test_emit
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
@@ -328,7 +333,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
 
   def test_emit_with_custom_index_name
     index_name = "logstash"
-    stub_request(:get, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
@@ -343,7 +348,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
 
   def test_emit_with_parse_timestamp
     index_name = "fluentd"
-    stub_request(:get, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
@@ -361,7 +366,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
 
   def test_emit_with_parse_timestamp_and_timstamp_format
     index_name = "fluentd"
-    stub_request(:get, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
                 headers: {'Content-Type' => 'application/json'})
@@ -380,7 +385,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
   end
 
   def test_emit_with_docinfo
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
@@ -399,11 +404,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
   end
 
   def test_emit_with_slices
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":0,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
@@ -419,12 +424,12 @@ class ElasticsearchInputTest < Test::Unit::TestCase
   end
 
   def test_emit_with_size
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_scroll_response.to_s,
                 headers: {'Content-Type' => 'application/json'})
     connection = 0
-    scroll_request = stub_request(:get, "http://localhost:9200/_search/scroll?scroll=1m").
+    scroll_request = stub_request(@http_method, "http://localhost:9200/_search/scroll?scroll=1m").
       with(
         body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}") do
       connection += 1