logstash-filter-elasticsearch 3.4.0 → 3.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92a2e2d6660ea8cf72381f7e7bd6832525e580c083b1f0d7188cbea58e32f2ee
-  data.tar.gz: 9cf2ac564ac0347e9f0642c0efaef8b4ac59f8725d9a9eef250805e8cde03656
+  metadata.gz: 57bc6bc825a1866b1ecf39eb35bcab5e062dfb73f7b1e7590feb4fe51358b2c9
+  data.tar.gz: 48df2bb55103e2fb6196ec8679f1b48ce70742f43dcd9ab3c5ff257637ccdda8
 SHA512:
-  metadata.gz: 913c2c225cfa517bf983ca3ee9c85263f0008e305b27dae4619afb8ffaa535e36f2618449b5da2de498dae4bfeb7e3f8dd2d654b3e0499d60f2383e99e92865b
-  data.tar.gz: b47a71caee22bf6b95908ec48ea9ce8a17b73c619fe48453a3ed4fe1470906bb149d4c852a1b8aba22e1b003ab0ad8ec33fcf558dfff56ea464cd9b69e4dc04e
+  metadata.gz: e93655c43a05df8d1f7cf261ff552a875cb765111903cf9bc4903e77cb423329f346f1b67d949b6cae24c85872b8ea45a7a08e7ce812d48ab93117d575236eef
+  data.tar.gz: b3dbca71043c7cbb6ef904e19e30b88dd6e974d92c0fdf77eae52d90c1d41151bdc9b0ef4c9cc93fa8c97937dc31e30c6570a62a8343d693a0b1d00bb2d4fa6a

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 3.5.0
+  - Added connection check during register to avoid failures during processing
+  - Changed Elasticsearch Client transport to use Manticore
+  - Changed amount of logging details during connection failure
+
 ## 3.4.0
   - Adds `[@metadata][total_hits]` with total hits returned from the query ([#106](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/106))
   - Improves error logging to fully inspect caught exceptions ([#105](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/105))

data/lib/logstash/filters/elasticsearch.rb

@@ -71,6 +71,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       @query_dsl = file.read
     end
 
+    test_connection!
   end # def register
 
   def filter(event)
@@ -123,7 +124,13 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       end
 
     rescue => e
-      @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e.inspect)
+      if @logger.trace?
+        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event.to_hash, :error => e.message, :backtrace => e.backtrace)
+      elsif @logger.debug?
+        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :error => e.message, :backtrace => e.backtrace)
+      else
+        @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :error => e.message)
+      end
       @tag_on_failure.each{|tag| event.tag(tag)}
     else
       filter_matched(event) if matched
@@ -165,4 +172,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       memo[old_key_fragment]
     end
   end
+
+  def test_connection!
+    get_client.client.ping
+  end
 end #class LogStash::Filters::Elasticsearch

data/lib/logstash/filters/elasticsearch/client.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "elasticsearch"
 require "base64"
+require "elasticsearch/transport/transport/http/manticore"
 
 
 module LogStash
@@ -22,10 +23,11 @@ module LogStash
 
         hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
         # set ca_file even if ssl isn't on, since the host can be an https url
-        transport_options[:ssl] = { ca_file: options[:ca_file] } if options[:ca_file]
+        ssl_options = { ssl: true, ca_file: options[:ca_file] } if options[:ca_file]
+        ssl_options ||= {}
 
         @logger.info("New ElasticSearch filter client", :hosts => hosts)
-        @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options)
+        @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options, transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, :ssl => ssl_options)
       end
 
       def search(params)

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.4.0'
+  s.version         = '3.5.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Copies fields from previous log events in Elasticsearch to current events "
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'elasticsearch', ">= 5.0.3", " <6.0.0"
+  s.add_runtime_dependency 'manticore', "~> 0.6"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/elasticsearch_spec.rb

@@ -9,6 +9,9 @@ describe LogStash::Filters::Elasticsearch do
   context "registration" do
 
     let(:plugin) { LogStash::Plugin.lookup("filter", "elasticsearch").new({}) }
+    before do
+      allow(plugin).to receive(:test_connection!)
+    end
 
     it "should not raise an exception" do
       expect {plugin.register}.to_not raise_error
@@ -37,6 +40,7 @@ describe LogStash::Filters::Elasticsearch do
     before(:each) do
       allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
       allow(client).to receive(:search).and_return(response)
+      allow(plugin).to receive(:test_connection!)
       plugin.register
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.5.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-09-14 00:00:00.000000000 Z
+date: 2018-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,6 +50,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 6.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: