logstash-filter-elasticsearch 3.1.6 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1d027cbce1e8a66c1fd88d31d60d7a9b4d3389ae
-  data.tar.gz: 18c348a0892415779a6b476c2699f5a15a1dd42e
+  metadata.gz: 364c2ed330ed87b3914f8a7644460ed48522b5db
+  data.tar.gz: a77e91ea3a1071c78742fc9ce972f0d5ee9791f0
 SHA512:
-  metadata.gz: 9b5dec688ea1fe123041d1da7dfa6109de38df87444edccc1b296cbb59b012561a8659eaa47b6e85c97f57e07ed8346a3eabf4930709671768988306514e6172
-  data.tar.gz: 29ef969be8fe60df16eb381d632791dc25a2ca3c3a5c676d290a617a300a2f3a6f30545c75b82abdc311f9c03358377afc53ff3eb9f8ed56193b924a7cd297a5
+  metadata.gz: c7df570123dc569cdffd64f310b154f735efe97306b93bd7c76ce8cc43a92f67b9558b16d27d99f653e454473dad2ddf3e95703ea16229625b043d8b5e723a4d
+  data.tar.gz: f64a717402ab19107450d6d800d3f5abb0ee32075b9c9afccdb61d8ba891b2f52f25a23c69117ecc0f816c42b42b8f9caa8e7e59b7fd44df5d1930cb58e1d04d

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.2.0
+  - `index` setting now supports field formatting, such as `index => "%{myindex}"` (Boris Gorbylev)
+
+## 3.1.8
+  - Fix a thread safety issue when using this filter with multiple workers on heavy load, we now create an elasticsearch client for every LogStash worker. #76
+
 ## 3.1.6
   - Fix some documentation issues
 

data/docs/index.asciidoc

@@ -163,7 +163,8 @@ List of elasticsearch hosts to use for querying.
   * Value type is <<string,string>>
   * Default value is `""`
 
-Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
+Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
+Field substitution (e.g. `index-name-%{date_field}`) is available
 
 [id="plugins-{type}s-{plugin}-password"]
 ===== `password` 

data/lib/logstash/filters/elasticsearch.rb

@@ -3,6 +3,7 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require_relative "elasticsearch/client"
 require "logstash/json"
+java_import "java.util.concurrent.ConcurrentHashMap"
 
 # .Compatibility Note
 # [NOTE]
@@ -87,7 +88,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # List of elasticsearch hosts to use for querying.
   config :hosts, :validate => :array,  :default => [ "localhost:9200" ]
   
-  # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
+  # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
+  # Field substitution (e.g. `index-name-%{date_field}`) is available
   config :index, :validate => :string, :default => ""
 
   # Elasticsearch query string. Read the Elasticsearch query string documentation.
@@ -125,14 +127,10 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Tags the event on failure to look up geo information. This can be used in later analysis.
   config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
 
+  attr_reader :clients_pool
+
   def register
-    options = {
-      :ssl => @ssl,
-      :hosts => @hosts,
-      :ca_file => @ca_file,
-      :logger => @logger
-    }
-    @client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options)
+    @clients_pool = java.util.concurrent.ConcurrentHashMap.new
 
     #Load query if it exists
     if @query_template
@@ -148,7 +146,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   def filter(event)
     begin
 
-      params = {:index => @index }
+      params = {:index => event.sprintf(@index) }
 
       if @query_dsl
         query = LogStash::Json.load(event.sprintf(@query_dsl))
@@ -162,7 +160,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
 
       @logger.debug("Querying elasticsearch for lookup", :params => params)
 
-      results = @client.search(params)
+      results = get_client.search(params)
       @fields.each do |old_key, new_key|
         if !results['hits']['hits'].empty?
           set = []
@@ -178,4 +176,22 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
     end
     filter_matched(event)
   end # def filter
+
+  private
+  def client_options
+    {
+      :ssl => @ssl,
+      :hosts => @hosts,
+      :ca_file => @ca_file,
+      :logger => @logger
+    }
+  end
+
+  def new_client
+    LogStash::Filters::ElasticsearchClient.new(@user, @password, client_options)
+  end
+
+  def get_client
+    @clients_pool.computeIfAbsent(Thread.current, lambda { |x| new_client })
+  end
 end #class LogStash::Filters::Elasticsearch

data/lib/logstash/filters/elasticsearch/client.rb

@@ -24,7 +24,7 @@ module LogStash
         # set ca_file even if ssl isn't on, since the host can be an https url
         transport_options[:ssl] = { ca_file: options[:ca_file] } if options[:ca_file]
 
-        @logger.info("New ElasticSearch filter", :hosts => hosts)
+        @logger.info("New ElasticSearch filter client", :hosts => hosts)
         @client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options)
       end
 

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.1.6'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -38,6 +38,25 @@ describe LogStash::Filters::Elasticsearch do
       plugin.register
     end
 
+    after(:each) do
+      Thread.current[:filter_elasticsearch_client] = nil
+    end
+
+    # Since the Elasticsearch Ruby client is not thread safe
+    # and under high load we can get error with the connection pool
+    # we have decided to create a new instance per worker thread which
+    # will be lazy created on the first call to `#filter`
+    #
+    # I am adding a simple test case for future changes
+    it "uses a different connection object per thread wait" do
+      expect(plugin.clients_pool.size).to eq(0)
+
+      Thread.new { plugin.filter(event) }.join
+      Thread.new { plugin.filter(event) }.join
+
+      expect(plugin.clients_pool.size).to eq(2)
+    end
+
     it "should enhance the current event with new data" do
       plugin.filter(event)
       expect(event.get("code")).to eq(404)
@@ -122,6 +141,30 @@ describe LogStash::Filters::Elasticsearch do
 
     end
 
+    context "testing a simple index substitution" do
+      let(:event) {
+        LogStash::Event.new(
+            {
+                "subst_field" => "subst_value"
+            }
+        )
+      }
+      let(:config) do
+        {
+            "index" => "foo_%{subst_field}*",
+            "hosts" => ["localhost:9200"],
+            "query" => "response: 404",
+            "fields" => [["response", "code"]]
+        }
+      end
+
+      it "should receive substituted index name" do
+        expect(client).to receive(:search).with({:q => "response: 404", :size => 1, :index => "foo_subst_value*", :sort => "@timestamp:desc"})
+        plugin.filter(event)
+      end
+
+    end
+
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.1.6
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-15 00:00:00.000000000 Z
+date: 2017-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement