logstash-input-elasticsearch 4.20.4 → 4.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8471702d33885c6c961b28b2008f5713140032b3599b0e31fbcc29a28e443828
-  data.tar.gz: 34921343e516dd0a21606d8ab33e270ead6db5ee0a3f93797ffea2916b736aeb
+  metadata.gz: bf89e42e7146cb59e938fed4a2a2d352bc72bf5c73c59d2a674be724c686a543
+  data.tar.gz: 9b0b0b80d4a3754dac6e5d4f2c179dbd1be2ee6d6ce6c2b1fc619c2369a808aa
 SHA512:
-  metadata.gz: ecf66d1369014d7d53dc0682b9a1174809c3fa1c571d13efb36c700fe1a9a819d5e1c13e21a2a13da2392e40b34b6bf5c9469ec431bbea2eed04e8547d9bae6e
-  data.tar.gz: db61b3448dec7659d304faafa163d81e0cf3b3a39f531a5638302ec75e69b7ef9e56741afed0c59552f95140c5cfb5641f5a506e9717802c423e43f028958a16
+  metadata.gz: 87223b2b656aee8d27c9f0ea92151370f3eb46b1851040054f0fa74e99ab4882278b28c85d4f8290d211ef793ab3eab9141b9c3ab343bd80cc5b9b9e7f0ceb11
+  data.tar.gz: 10db333c4a67f69fb695bede43feb8f89692609afd74c40de3babf9b56580c18ea1d839865c66e9ec5ade759825309aecd84a2d269805583e8053b48bb563af9

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.21.0
+  - Add support for custom headers [#217](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/217)
+
+## 4.20.5
+  - Add `x-elastic-product-origin` header to Elasticsearch requests [#211](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/211)
+
 ## 4.20.4
   - Fix issue where the `index` parameter was being ignored when using `response_type => aggregations` [#209](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/209)
 

data/docs/index.asciidoc

@@ -23,7 +23,7 @@ include::{include_path}/plugin_header.asciidoc[]
 
 Read from an Elasticsearch cluster, based on search query results.
 This is useful for replaying test logs, reindexing, etc.
-You can periodically schedule ingestion using a cron syntax 
+You can periodically schedule ingestion using a cron syntax
 (see `schedule` setting) or run the query one time to load
 data into Logstash.
 
@@ -106,6 +106,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
+| <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
@@ -199,8 +200,18 @@ For more info, check out the
 The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch.
 Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool.
 
+[id="plugins-{type}s-{plugin}-custom_headers"]
+===== `custom_headers`
+
+  * Value type is <<hash,hash>>
+  * Default value is empty
+
+Pass a set of key value pairs as the headers sent in each request to an elasticsearch node.
+The headers will be used for any kind of request.
+These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers.
+
 [id="plugins-{type}s-{plugin}-docinfo"]
-===== `docinfo` 
+===== `docinfo`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -251,7 +262,7 @@ Example
 
 
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
-===== `docinfo_fields` 
+===== `docinfo_fields`
 
   * Value type is <<array,array>>
   * Default value is `["_index", "_type", "_id"]`
@@ -262,7 +273,7 @@ option lists the metadata fields to save in the current event. See
 more information.
 
 [id="plugins-{type}s-{plugin}-docinfo_target"]
-===== `docinfo_target` 
+===== `docinfo_target`
 
   * Value type is <<string,string>>
   * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
@@ -286,7 +297,7 @@ this option names the field under which to store the metadata fields as subfield
 Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
 
 [id="plugins-{type}s-{plugin}-hosts"]
-===== `hosts` 
+===== `hosts`
 
   * Value type is <<array,array>>
   * There is no default value for this setting.
@@ -296,18 +307,18 @@ can be either IP, HOST, IP:port, or HOST:port. The port defaults to
 9200.
 
 [id="plugins-{type}s-{plugin}-index"]
-===== `index` 
+===== `index`
 
   * Value type is <<string,string>>
   * Default value is `"logstash-*"`
 
-The index or alias to search. 
+The index or alias to search.
 Check out {ref}/api-conventions.html#api-multi-index[Multi Indices
 documentation] in the Elasticsearch documentation for info on
 referencing multiple indices.
 
 [id="plugins-{type}s-{plugin}-password"]
-===== `password` 
+===== `password`
 
   * Value type is <<password,password>>
   * There is no default value for this setting.
@@ -327,7 +338,7 @@ An empty string is treated as if proxy was not set, this is useful when using
 environment variables e.g. `proxy => '${LS_PROXY:}'`.
 
 [id="plugins-{type}s-{plugin}-query"]
-===== `query` 
+===== `query`
 
   * Value type is <<string,string>>
   * Default value is `'{ "sort": [ "_doc" ] }'`
@@ -375,7 +386,7 @@ The default is 0 (no retry). This value should be equal to or greater than zero.
 NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible.
 
 [id="plugins-{type}s-{plugin}-schedule"]
-===== `schedule` 
+===== `schedule`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -387,7 +398,7 @@ There is no schedule by default. If no schedule is given, then the statement is
 exactly once.
 
 [id="plugins-{type}s-{plugin}-scroll"]
-===== `scroll` 
+===== `scroll`
 
   * Value type is <<string,string>>
   * Default value is `"1m"`
@@ -410,7 +421,7 @@ The query requires at least one `sort` field, as described in the <<plugins-{typ
 `scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended.
 
 [id="plugins-{type}s-{plugin}-size"]
-===== `size` 
+===== `size`
 
   * Value type is <<number,number>>
   * Default value is `1000`
@@ -598,7 +609,7 @@ It is also possible to target an entry in the event's metadata, which will be av
 
 
 [id="plugins-{type}s-{plugin}-user"]
-===== `user` 
+===== `user`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.

data/lib/logstash/inputs/elasticsearch.rb

@@ -24,9 +24,9 @@ require_relative "elasticsearch/patches/_elasticsearch_transport_connections_sel
 # called `http.content_type.required`. If this option is set to `true`, and you
 # are using Logstash 2.4 through 5.2, you need to update the Elasticsearch input
 # plugin to version 4.0.2 or higher.
-# 
+#
 # ================================================================================
-# 
+#
 # Read from an Elasticsearch cluster, based on search query results.
 # This is useful for replaying test logs, reindexing, etc.
 # It also supports periodically scheduling lookup enrichments
@@ -166,6 +166,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html
   config :docinfo_fields, :validate => :array, :default => ['_index', '_type', '_id']
 
+  # Custom headers for Elasticsearch requests
+  config :custom_headers, :validate => :hash, :default => {}
+
   # Basic Auth - username
   config :user, :validate => :string
 
@@ -271,6 +274,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
   BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
   DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
+  INTERNAL_ORIGIN_HEADER = { 'x-elastic-product-origin' => 'logstash-input-elasticsearch'}.freeze
 
   def initialize(params={})
     super(params)
@@ -300,9 +304,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     fill_user_password_from_cloud_auth
 
     transport_options = {:headers => {}}
+    transport_options[:headers].merge!(INTERNAL_ORIGIN_HEADER)
     transport_options[:headers].merge!(setup_basic_auth(user, password))
     transport_options[:headers].merge!(setup_api_key(api_key))
     transport_options[:headers].merge!({'user-agent' => prepare_user_agent()})
+    transport_options[:headers].merge!(@custom_headers) unless @custom_headers.empty?
     transport_options[:request_timeout] = @request_timeout_seconds unless @request_timeout_seconds.nil?
     transport_options[:connect_timeout] = @connect_timeout_seconds unless @connect_timeout_seconds.nil?
     transport_options[:socket_timeout]  = @socket_timeout_seconds  unless @socket_timeout_seconds.nil?

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.20.4'
+  s.version         = '4.21.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/elasticsearch_spec.rb

@@ -50,6 +50,12 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
         client = plugin.send(:client)
         expect( extract_transport(client).options[:transport_options][:headers] ).not_to match hash_including("Elastic-Api-Version" => "2023-10-31")
       end
+
+      it "sets an x-elastic-product-origin header identifying this as an internal plugin request" do
+        plugin.register
+        client = plugin.send(:client)
+        expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("x-elastic-product-origin"=>"logstash-input-elasticsearch")
+      end
     end
 
     context "against not authentic Elasticsearch" do
@@ -90,6 +96,28 @@ describe LogStash::Inputs::Elasticsearch, :ecs_compatibility_support do
           client = plugin.send(:client)
           expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("Elastic-Api-Version" => "2023-10-31")
         end
+
+        it "sets an x-elastic-product-origin header identifying this as an internal plugin request" do
+          plugin.register
+          client = plugin.send(:client)
+          expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("x-elastic-product-origin"=>"logstash-input-elasticsearch")
+        end
+      end
+
+      context "with custom headers" do
+        let(:config) do
+          {
+            "schedule" => "* * * * * UTC",
+            "custom_headers" => { "Custom-Header-1" => "Custom Value 1", "Custom-Header-2" => "Custom Value 2" }
+          }
+        end
+
+
+        it "sets custom headers" do
+          plugin.register
+          client = plugin.send(:client)
+          expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including(config["custom_headers"])
+        end
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.20.4
+  version: 4.21.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-23 00:00:00.000000000 Z
+date: 2024-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -37,8 +37,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.3'
   name: logstash-mixin-ecs_compatibility_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-event_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -65,8 +65,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-validator_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -79,8 +79,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-scheduler
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -93,8 +93,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 7.17.9
   name: elasticsearch
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -107,8 +107,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-ca_trusted_fingerprint_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -121,8 +121,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-normalize_config_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -135,8 +135,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: tzinfo
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -149,8 +149,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: tzinfo-data
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -163,8 +163,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.7.1
   name: manticore
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -177,8 +177,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -191,8 +191,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -205,8 +205,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: timecop
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -219,8 +219,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
   name: cabin
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -233,8 +233,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: webrick
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -247,8 +247,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.0.9
   name: rufus-scheduler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -307,7 +307,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster