logstash-filter-elasticsearch 3.0.2 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f50e0c72955c05e7b02de305d9dbf1fcc5da9707
4
- data.tar.gz: 86c11d6674eb5438c08f8899e8673f41c0e4d6c0
3
+ metadata.gz: 8ff652465fd4904e101402ef5c0085fa249859e6
4
+ data.tar.gz: 594e66f78a3c9676391b9d58e2e3d1eb6fb5cf8a
5
5
  SHA512:
6
- metadata.gz: e8205d6fab8a45d604f440bb076f27af94a775908dd7fc52d6418bc29fc33376ed506339251526c0b76d5185fcfd59840c72b1592eddcf8e1d04d89771bb5fa3
7
- data.tar.gz: 9636372242cf3a70d50c6cbb5d3032254b243c91c38e6f67c8301bc8d925950dd0bfc95b6678169d69ff468c196cec278e724d51f334085f4c1bf8031331aa19
6
+ metadata.gz: 1a48fb3287a46b475ce29dd03e2982c1a246b031f4f45fb951606f5819846b76bd6a28c1392a0c2e836f29eef086105cce715849eee35d634d546769c2c04b25
7
+ data.tar.gz: f2ee559904b31e25b7c0b8f6ed212f096babe29d5b11637a88f222522a93566d98c71d947d735b2bab96fc6deefa2a799f5540ef40c9790863b6073348987eeb
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ ## 3.1.0
2
+ - Support for full use of query DSL. Added query_template to use full DSL.
3
+
4
+ ## 3.0.3
5
+ - Fix couple of bugs related to incorrect variable names
6
+
1
7
  ## 3.0.2
2
8
  - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
3
9
 
@@ -2,21 +2,25 @@
2
2
  require "logstash/filters/base"
3
3
  require "logstash/namespace"
4
4
  require_relative "elasticsearch/client"
5
+ require "logstash/json"
5
6
 
6
-
7
- # Search elasticsearch for a previous log event and copy some fields from it
8
- # into the current event. Below is a complete example of how this filter might
9
- # be used. Whenever logstash receives an "end" event, it uses this elasticsearch
7
+ # Search Elasticsearch for a previous log event and copy some fields from it
8
+ # into the current event. Below are two complete examples of how this filter might
9
+ # be used.
10
+ #
11
+ # The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
12
+ # Whenever logstash receives an "end" event, it uses this elasticsearch
10
13
  # filter to find the matching "start" event based on some operation identifier.
11
14
  # Then it copies the `@timestamp` field from the "start" event into a new field on
12
15
  # the "end" event. Finally, using a combination of the "date" filter and the
13
16
  # "ruby" filter, we calculate the time duration in hours between the two events.
14
17
  # [source,ruby]
18
+ # --------------------------------------------------
15
19
  # if [type] == "end" {
16
20
  # elasticsearch {
17
21
  # hosts => ["es-server"]
18
22
  # query => "type:start AND operation:%{[opid]}"
19
- # fields => [["@timestamp", "started"]]
23
+ # fields => { "@timestamp" => "started" }
20
24
  # }
21
25
  #
22
26
  # date {
@@ -29,6 +33,44 @@ require_relative "elasticsearch/client"
29
33
  # }
30
34
  # }
31
35
  #
36
+ # The example below reproduces the above example but utilises the query_template. This query_template represents a full
37
+ # Elasticsearch query DSL and supports the standard Logstash field substitution syntax. The example below issues
38
+ # the same query as the first example but uses the template shown.
39
+ #
40
+ # if [type] == "end" {
41
+ # elasticsearch {
42
+ # hosts => ["es-server"]
43
+ # query_template => "template.json"
44
+ # }
45
+ #
46
+ # date {
47
+ # match => ["[started]", "ISO8601"]
48
+ # target => "[started]"
49
+ # }
50
+ #
51
+ # ruby {
52
+ # code => "event['duration_hrs'] = (event['@timestamp'] - event['started']) / 3600 rescue nil"
53
+ # }
54
+ # }
55
+ #
56
+ #
57
+ #
58
+ # template.json:
59
+ #
60
+ # {
61
+ # "query": {
62
+ # "query_string": {
63
+ # "query": "type:start AND operation:%{[opid]}"
64
+ # }
65
+ # },
66
+ # "_source": ["@timestamp", "started"]
67
+ # }
68
+ #
69
+ # As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
70
+ # The template will be populated per event prior to being used to query Elasticsearch.
71
+ #
72
+ # --------------------------------------------------
73
+
32
74
  class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
33
75
  config_name "elasticsearch"
34
76
 
@@ -38,10 +80,14 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
38
80
  # Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
39
81
  config :index, :validate => :string, :default => ""
40
82
 
41
- # Elasticsearch query string. Read the Elasticsearch query string documentation
83
+ # Elasticsearch query string. Read the Elasticsearch query string documentation.
42
84
  # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
43
85
  config :query, :validate => :string
44
86
 
87
+ # File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
88
+ # for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
89
+ config :query_template, :validate => :string
90
+
45
91
  # Comma-delimited list of `<field>:<direction>` pairs that define the sort order
46
92
  config :sort, :validate => :string, :default => "@timestamp:desc"
47
93
 
@@ -77,16 +123,36 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
77
123
  :logger => @logger
78
124
  }
79
125
  @client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options)
126
+
127
+ #Load query if it exists
128
+ if @query_template
129
+ if File.zero?(@query_template)
130
+ raise "template is empty"
131
+ end
132
+ file = File.open(@query_template, "rb")
133
+ @query_dsl = file.read
134
+ end
135
+
80
136
  end # def register
81
137
 
82
138
  def filter(event)
83
139
  begin
84
- query_str = event.sprintf(@query)
85
140
 
86
- params = { :q => query_str, :size => result_size, :index => @index }
87
- params[:sort] = @sort if @enable_sort
88
- results = @client.search(params)
141
+ params = {:index => @index }
89
142
 
143
+ if @query_dsl
144
+ query = LogStash::Json.load(event.sprintf(@query_dsl))
145
+ params[:body] = query
146
+ else
147
+ query = event.sprintf(@query)
148
+ params[:q] = query
149
+ params[:size] = result_size
150
+ params[:sort] = @sort if @enable_sort
151
+ end
152
+
153
+ @logger.info("Querying elasticsearch for lookup", :params => params)
154
+
155
+ results = @client.search(params)
90
156
  @fields.each do |old_key, new_key|
91
157
  if !results['hits']['hits'].empty?
92
158
  set = []
@@ -97,7 +163,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
97
163
  end
98
164
  end
99
165
  rescue => e
100
- @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query_str, :event => event, :error => e)
166
+ @logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e)
101
167
  @tag_on_failure.each{|tag| event.tag(tag)}
102
168
  end
103
169
  filter_matched(event)
@@ -2,6 +2,7 @@
2
2
  require "elasticsearch"
3
3
  require "base64"
4
4
 
5
+
5
6
  module LogStash
6
7
  module Filters
7
8
  class ElasticsearchClient
@@ -9,7 +10,7 @@ module LogStash
9
10
  attr_reader :client
10
11
 
11
12
  def initialize(user, password, options={})
12
- ssl = options.fetch(:ssh, false)
13
+ ssl = options.fetch(:ssl, false)
13
14
  hosts = options[:hosts]
14
15
  @logger = options[:logger]
15
16
 
@@ -19,7 +20,7 @@ module LogStash
19
20
  transport_options[:headers] = { Authorization: "Basic #{token}" }
20
21
  end
21
22
 
22
- host.map! {|h| { host: h, scheme: 'https' } } if ssl
23
+ hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
23
24
  transport_options[:ssl] = { ca_file: options[:ca_file] } if ssl && options[:ca_file]
24
25
 
25
26
  @logger.info("New ElasticSearch filter", :hosts => hosts)
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-elasticsearch'
4
- s.version = '3.0.2'
4
+ s.version = '3.1.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -100,6 +100,28 @@ describe LogStash::Filters::Elasticsearch do
100
100
  expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
101
101
  end
102
102
  end
103
+
104
+ context "testing a simple query template" do
105
+ let(:config) do
106
+ {
107
+ "hosts" => ["localhost:9200"],
108
+ "query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
109
+ "fields" => [ ["response", "code"] ],
110
+ "result_size" => 1
111
+ }
112
+ end
113
+
114
+ let(:response) do
115
+ LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
116
+ end
117
+
118
+ it "should enhance the current event with new data" do
119
+ plugin.filter(event)
120
+ expect(event.get("code")).to eq(404)
121
+ end
122
+
123
+ end
124
+
103
125
  end
104
126
 
105
127
  end
@@ -0,0 +1,7 @@
1
+ {
2
+ "query": {
3
+ "query_string": {
4
+ "query": "response: 404"
5
+ }
6
+ }
7
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.2
4
+ version: 3.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-14 00:00:00.000000000 Z
11
+ date: 2016-12-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -74,6 +74,7 @@ files:
74
74
  - lib/logstash/filters/elasticsearch/client.rb
75
75
  - logstash-filter-elasticsearch.gemspec
76
76
  - spec/filters/elasticsearch_spec.rb
77
+ - spec/filters/fixtures/query_template.json
77
78
  - spec/filters/fixtures/request_x_1.json
78
79
  - spec/filters/fixtures/request_x_10.json
79
80
  - spec/filters/integration/elasticsearch_spec.rb
@@ -99,12 +100,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
99
100
  version: '0'
100
101
  requirements: []
101
102
  rubyforge_project:
102
- rubygems_version: 2.6.3
103
+ rubygems_version: 2.4.8
103
104
  signing_key:
104
105
  specification_version: 4
105
106
  summary: Search elasticsearch for a previous log event and copy some fields from it into the current event
106
107
  test_files:
107
108
  - spec/filters/elasticsearch_spec.rb
109
+ - spec/filters/fixtures/query_template.json
108
110
  - spec/filters/fixtures/request_x_1.json
109
111
  - spec/filters/fixtures/request_x_10.json
110
112
  - spec/filters/integration/elasticsearch_spec.rb