logstash-input-elasticsearch 4.8.1 → 4.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2739ba5eed141d75ad674402bb35485e0be50c6a7593556dbcda575120bbb54
-  data.tar.gz: 1affc8bfa4aad83cda6c59f0a6b3dcf7ca24a5d7e0e53cacc66f45b88d353ce1
+  metadata.gz: 4091feeb0b3bf292cfb9afcde7496a72f95168eb570b21d29064ade174bb1352
+  data.tar.gz: cfd02af050bb495dceea16b4ffe5daa6828088d2bd7994639ee08374f87da69d
 SHA512:
-  metadata.gz: d4d2eb3be415ddf52ed7b17d06999fe93022004fcf218d11919c750e3b98f8a1ddc943d7d0911232a43fcc8f237a08ffcbd63ba5ec1dd59de9498061f4b4bf25
-  data.tar.gz: 74524dcd7cacd49a324049f80b5cabf5bd2a96c8c634ed15877139371cc8cb89bb38c074a576a0214a3785e218ee5dc0f0966b59de75394e679d8eef34ad14ed
+  metadata.gz: 873680bea22204e65d519d310f3952f26fd672ce336504e45e99b13988e2da2794b7c05f83b3e1b7a87f317eb51c079759f9df515dc99236577417ecfd379aa1
+  data.tar.gz: 85c66a665eb7a3b503ab7cec64ea2f24b0e16829c4b78a75560d69c6272b1861583dad41e04ea1157ce58435714a86eaa6f4d71b5a4a724edfb9604f77150b99

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 4.9.0
+  - Added `target` option, allowing the hit's source to target a specific field instead of being expanded at the root of the event. This allows the input to play nicer with the Elastic Common Schema when the input does not follow the schema. [#117](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/117)
+
+## 4.8.3
+  - [DOC] Fixed links to restructured Logstash-to-cloud docs [#139](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/139)
+
+## 4.8.2
+  - [DOC] Document the permissions required in secured clusters [#137](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/137)
+
 ## 4.8.1
   - Fixed connection error when using multiple `slices`. [#133](https://github.com/logstash-plugins/logstash-input-elasticsearch/issues/133)
 

data/CONTRIBUTORS

@@ -7,6 +7,7 @@ Contributors:
 * Jordan Sissel (jordansissel)
 * João Duarte (jsvd)
 * Kurt Hurtado (kurtado)
+* Luca Belluccini (lucabelluccini)
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)

data/docs/index.asciidoc

@@ -77,6 +77,11 @@ Authentication to a secure Elasticsearch cluster is possible using _one_ of the
 * <<plugins-{type}s-{plugin}-cloud_auth>>
 * <<plugins-{type}s-{plugin}-api_key>>
 
+[id="plugins-{type}s-{plugin}-autz"]
+==== Authorization
+
+Authorization to a secure Elasticsearch cluster requires `read` permission at index level and `monitoring` permissions at cluster level.
+The `monitoring` permission at cluster level is necessary to perform periodic connectivity checks.
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input Configuration Options
@@ -106,6 +111,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-slices>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-socket_timeout_seconds>> | <<number,number>>|No
+| <<plugins-{type}s-{plugin}-target>> | https://www.elastic.co/guide/en/logstash/master/field-references-deepdive.html[field reference] | No
 | <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 |=======================================================================
 
@@ -122,7 +128,10 @@ input plugins.
 
 Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option.
 
-Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key API].
+Format is `id:api_key` where `id` and `api_key` are as returned by the
+Elasticsearch
+https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create
+API key API].
 
 [id="plugins-{type}s-{plugin}-ca_file"]
 ===== `ca_file` 
@@ -140,7 +149,9 @@ SSL Certificate Authority file in PEM encoded format, must also include any chai
 
 Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
 
-For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
+For more info, check out the
+https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html[Logstash-to-Cloud
+documentation]
 
 [id="plugins-{type}s-{plugin}-cloud_id"]
 ===== `cloud_id`
@@ -150,7 +161,9 @@ For more info, check out the https://www.elastic.co/guide/en/logstash/current/co
 
 Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 
-For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
+For more info, check out the
+https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html[Logstash-to-Cloud
+documentation]
 
 [id="plugins-{type}s-{plugin}-connect_timeout_seconds"]
 ===== `connect_timeout_seconds`
@@ -365,6 +378,19 @@ server (i.e. HTTPS will be used instead of plain HTTP).
 The maximum amount of time, in seconds, to wait on an incomplete response from Elasticsearch while no additional data has been appended.
 Socket timeouts usually occur while waiting for the first byte of a response, such as when executing a particularly complex query.
 
+
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+* Value type is https://www.elastic.co/guide/en/logstash/master/field-references-deepdive.html[field reference]
+* There is no default value for this setting.
+
+Without a `target`, events are created from each hit's `_source` at the root level.
+When the `target` is set to a field reference, the `_source` of the hit is placed in the target field instead.
+
+This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced.
+It is also possible to target an entry in the event's metadata, which will be available during event processing but not exported to your outputs (e.g., `target \=> "[@metadata][_source]"`).
+
 [id="plugins-{type}s-{plugin}-user"]
 ===== `user` 
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -3,6 +3,7 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/json"
 require "logstash/util/safe_uri"
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require "base64"
 require_relative "patch"
 
@@ -62,6 +63,8 @@ require_relative "patch"
 #
 #
 class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "elasticsearch"
 
   default :codec, "json"
@@ -175,6 +178,9 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # exactly once.
   config :schedule, :validate => :string
 
+  # If set, the _source of each hit will be added nested under the target instead of at the top-level
+  config :target, :validate => :field_reference
+
   def register
     require "elasticsearch"
     require "rufus/scheduler"
@@ -298,7 +304,12 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   end
 
   def push_hit(hit, output_queue)
-    event = LogStash::Event.new(hit['_source'])
+    if @target.nil?
+      event = LogStash::Event.new(hit['_source'])
+    else
+      event = LogStash::Event.new
+      event.set(@target, hit['_source'])
+    end
 
     if @docinfo
       # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.8.1'
+  s.version         = '4.9.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
+  s.add_runtime_dependency "logstash-mixin-validator_support", '~> 1.0'
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   s.add_runtime_dependency 'elasticsearch', '>= 5.0.3'

data/spec/inputs/elasticsearch_spec.rb

@@ -40,57 +40,91 @@ describe LogStash::Inputs::TestableElasticsearch do
     end
   end
 
-  it "should retrieve json event from elasticseach" do
-    config = %q[
-      input {
-        elasticsearch {
-          hosts => ["localhost"]
-          query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+  context 'creating events from Elasticsearch' do
+    let(:config) do
+      %q[
+        input {
+          elasticsearch {
+            hosts => ["localhost"]
+            query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+          }
         }
+      ]
+    end
+
+    let(:mock_response) do
+      {
+          "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+          "took" => 27,
+          "timed_out" => false,
+          "_shards" => {
+              "total" => 169,
+              "successful" => 169,
+              "failed" => 0
+          },
+          "hits" => {
+              "total" => 1,
+              "max_score" => 1.0,
+              "hits" => [ {
+                              "_index" => "logstash-2014.10.12",
+                              "_type" => "logs",
+                              "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+                              "_score" => 1.0,
+                              "_source" => { "message" => ["ohayo"] }
+                          } ]
+          }
       }
-    ]
-
-    response = {
-      "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
-      "took" => 27,
-      "timed_out" => false,
-      "_shards" => {
-        "total" => 169,
-        "successful" => 169,
-        "failed" => 0
-      },
-      "hits" => {
-        "total" => 1,
-        "max_score" => 1.0,
-        "hits" => [ {
-          "_index" => "logstash-2014.10.12",
-          "_type" => "logs",
-          "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
-          "_score" => 1.0,
-          "_source" => { "message" => ["ohayo"] }
-        } ]
+    end
+
+    let(:mock_scroll_response) do
+      {
+          "_scroll_id" => "r453Wc1jh0caLJhSDg",
+          "hits" => { "hits" => [] }
       }
-    }
+    end
 
-    scroll_reponse = {
-      "_scroll_id" => "r453Wc1jh0caLJhSDg",
-      "hits" => { "hits" => [] }
-    }
+    before(:each) do
+      client = Elasticsearch::Client.new
+      expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
+      expect(client).to receive(:search).with(any_args).and_return(mock_response)
+      expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(mock_scroll_response)
+      expect(client).to receive(:clear_scroll).and_return(nil)
+    end
 
-    client = Elasticsearch::Client.new
-    expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
-    expect(client).to receive(:search).with(any_args).and_return(response)
-    expect(client).to receive(:scroll).with({ :body => { :scroll_id => "cXVlcnlUaGVuRmV0Y2g" }, :scroll=> "1m" }).and_return(scroll_reponse)
-    expect(client).to receive(:clear_scroll).and_return(nil)
+    it 'creates the events from the hits' do
+      event = input(config) do |pipeline, queue|
+        queue.pop
+      end
 
-    event = input(config) do |pipeline, queue|
-      queue.pop
+      expect(event).to be_a(LogStash::Event)
+      puts event.to_hash_with_metadata
+      expect(event.get("message")).to eql [ "ohayo" ]
     end
 
-    expect(event).to be_a(LogStash::Event)
-    expect(event.get("message")).to eql [ "ohayo" ]
-  end
+    context 'when a target is set' do
+      let(:config) do
+        %q[
+          input {
+            elasticsearch {
+              hosts => ["localhost"]
+              query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
+              target => "[@metadata][_source]"
+            }
+          }
+        ]
+      end
 
+      it 'creates the event using the target' do
+        event = input(config) do |pipeline, queue|
+          queue.pop
+        end
+
+        expect(event).to be_a(LogStash::Event)
+        puts event.to_hash_with_metadata
+        expect(event.get("[@metadata][_source][message]")).to eql [ "ohayo" ]
+      end
+    end
+  end
 
   # This spec is an adapter-spec, ensuring that we send the right sequence of messages to our Elasticsearch Client
   # to support sliced scrolling. The underlying implementation will spawn its own threads to consume, so we must be

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.8.1
+  version: 4.9.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-20 00:00:00.000000000 Z
+date: 2020-12-16 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -231,7 +245,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster