logstash-filter-elasticsearch 3.0.2 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f50e0c72955c05e7b02de305d9dbf1fcc5da9707
-  data.tar.gz: 86c11d6674eb5438c08f8899e8673f41c0e4d6c0
+  metadata.gz: 8ff652465fd4904e101402ef5c0085fa249859e6
+  data.tar.gz: 594e66f78a3c9676391b9d58e2e3d1eb6fb5cf8a
 SHA512:
-  metadata.gz: e8205d6fab8a45d604f440bb076f27af94a775908dd7fc52d6418bc29fc33376ed506339251526c0b76d5185fcfd59840c72b1592eddcf8e1d04d89771bb5fa3
-  data.tar.gz: 9636372242cf3a70d50c6cbb5d3032254b243c91c38e6f67c8301bc8d925950dd0bfc95b6678169d69ff468c196cec278e724d51f334085f4c1bf8031331aa19
+  metadata.gz: 1a48fb3287a46b475ce29dd03e2982c1a246b031f4f45fb951606f5819846b76bd6a28c1392a0c2e836f29eef086105cce715849eee35d634d546769c2c04b25
+  data.tar.gz: f2ee559904b31e25b7c0b8f6ed212f096babe29d5b11637a88f222522a93566d98c71d947d735b2bab96fc6deefa2a799f5540ef40c9790863b6073348987eeb

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.1.0
+  - Support for full use of query DSL. Added query_template to use full DSL.
+
+## 3.0.3
+  - Fix couple of bugs related to incorrect variable names
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/lib/logstash/filters/elasticsearch.rb

@@ -2,21 +2,25 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require_relative "elasticsearch/client"
+require "logstash/json"
 
-
-# Search elasticsearch for a previous log event and copy some fields from it
-# into the current event.  Below is a complete example of how this filter might
-# be used.  Whenever logstash receives an "end" event, it uses this elasticsearch
+# Search Elasticsearch for a previous log event and copy some fields from it
+# into the current event.  Below are two complete examples of how this filter might
+# be used.
+#
+# The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
+# Whenever logstash receives an "end" event, it uses this elasticsearch
 # filter to find the matching "start" event based on some operation identifier.
 # Then it copies the `@timestamp` field from the "start" event into a new field on
 # the "end" event.  Finally, using a combination of the "date" filter and the
 # "ruby" filter, we calculate the time duration in hours between the two events.
 # [source,ruby]
+# --------------------------------------------------
 #       if [type] == "end" {
 #          elasticsearch {
 #             hosts => ["es-server"]
 #             query => "type:start AND operation:%{[opid]}"
-#             fields => [["@timestamp", "started"]]
+#             fields => { "@timestamp" => "started" }
 #          }
 #
 #          date {
@@ -29,6 +33,44 @@ require_relative "elasticsearch/client"
 #          }
 #       }
 #
+#  The example below reproduces the above example but utilises the query_template.  This query_template represents a full
+#  Elasticsearch query DSL and supports the standard Logstash field substitution syntax.  The example below issues
+#  the same query as the first example but uses the template shown.
+#
+#   if [type] == "end" {
+#          elasticsearch {
+#             hosts => ["es-server"]
+#             query_template => "template.json"
+#          }
+#
+#          date {
+#             match => ["[started]", "ISO8601"]
+#             target => "[started]"
+#          }
+#
+#          ruby {
+#             code => "event['duration_hrs'] = (event['@timestamp'] - event['started']) / 3600 rescue nil"
+#          }
+#   }
+#
+#
+#
+#   template.json:
+#
+#  {
+#     "query": {
+#       "query_string": {
+#        "query": "type:start AND operation:%{[opid]}"
+#       }
+#     },
+#    "_source": ["@timestamp", "started"]
+#  }
+#
+# As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
+# The template will be populated per event prior to being used to query Elasticsearch.
+#
+# --------------------------------------------------
+
 class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   config_name "elasticsearch"
 
@@ -38,10 +80,14 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
   # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
   config :index, :validate => :string, :default => ""
 
-  # Elasticsearch query string. Read the Elasticsearch query string documentation
+  # Elasticsearch query string. Read the Elasticsearch query string documentation.
   # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
   config :query, :validate => :string
 
+  # File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
+  # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
+  config :query_template, :validate => :string
+
   # Comma-delimited list of `<field>:<direction>` pairs that define the sort order
   config :sort, :validate => :string, :default => "@timestamp:desc"
 
@@ -77,16 +123,36 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
       :logger => @logger
     }
     @client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options)
+
+    #Load query if it exists
+    if @query_template
+      if File.zero?(@query_template)
+        raise "template is empty"
+      end
+      file = File.open(@query_template, "rb")
+      @query_dsl = file.read
+    end
+
   end # def register
 
   def filter(event)
     begin
-      query_str = event.sprintf(@query)
 
-      params = { :q => query_str, :size => result_size, :index => @index }
-      params[:sort] =  @sort if @enable_sort
-      results = @client.search(params)
+      params = {:index => @index }
 
+      if @query_dsl
+        query = LogStash::Json.load(event.sprintf(@query_dsl))
+        params[:body] = query
+      else
+        query = event.sprintf(@query)
+        params[:q] = query
+        params[:size] = result_size
+        params[:sort] =  @sort if @enable_sort
+      end
+
+      @logger.info("Querying elasticsearch for lookup", :params => params)
+
+      results = @client.search(params)
       @fields.each do |old_key, new_key|
         if !results['hits']['hits'].empty?
           set = []
@@ -97,7 +163,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
         end
       end
     rescue => e
-      @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query_str, :event => event, :error => e)
+      @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e)
       @tag_on_failure.each{|tag| event.tag(tag)}
     end
     filter_matched(event)

data/lib/logstash/filters/elasticsearch/client.rb

@@ -2,6 +2,7 @@
 require "elasticsearch"
 require "base64"
 
+
 module LogStash
   module Filters
     class ElasticsearchClient
@@ -9,7 +10,7 @@ module LogStash
       attr_reader :client
 
       def initialize(user, password, options={})
-        ssl     = options.fetch(:ssh, false)
+        ssl     = options.fetch(:ssl, false)
         hosts   = options[:hosts]
         @logger = options[:logger]
 
@@ -19,7 +20,7 @@ module LogStash
           transport_options[:headers] = { Authorization: "Basic #{token}" }
         end
 
-        host.map! {|h| { host: h, scheme: 'https' } } if ssl
+        hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
         transport_options[:ssl] = { ca_file: options[:ca_file] } if ssl && options[:ca_file]
 
         @logger.info("New ElasticSearch filter", :hosts => hosts)

data/logstash-filter-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-elasticsearch'
-  s.version         = '3.0.2'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/elasticsearch_spec.rb

@@ -100,6 +100,28 @@ describe LogStash::Filters::Elasticsearch do
         expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
       end
     end
+
+    context "testing a simple query template" do
+      let(:config) do
+        {
+            "hosts" => ["localhost:9200"],
+            "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
+            "fields" => [ ["response", "code"] ],
+            "result_size" => 1
+        }
+      end
+
+      let(:response) do
+        LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
+      end
+
+      it "should enhance the current event with new data" do
+        plugin.filter(event)
+        expect(event.get("code")).to eq(404)
+      end
+
+    end
+
   end
 
 end

data/spec/filters/fixtures/query_template.json

@@ -0,0 +1,7 @@
+{
+  "query": {
+    "query_string": {
+      "query": "response: 404"
+    }
+  }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 3.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -74,6 +74,7 @@ files:
 - lib/logstash/filters/elasticsearch/client.rb
 - logstash-filter-elasticsearch.gemspec
 - spec/filters/elasticsearch_spec.rb
+- spec/filters/fixtures/query_template.json
 - spec/filters/fixtures/request_x_1.json
 - spec/filters/fixtures/request_x_10.json
 - spec/filters/integration/elasticsearch_spec.rb
@@ -99,12 +100,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Search elasticsearch for a previous log event and copy some fields from it into the current event
 test_files:
 - spec/filters/elasticsearch_spec.rb
+- spec/filters/fixtures/query_template.json
 - spec/filters/fixtures/request_x_1.json
 - spec/filters/fixtures/request_x_10.json
 - spec/filters/integration/elasticsearch_spec.rb