logstash-filter-elasticsearch 3.2.1 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/LICENSE +1 -1
- data/docs/index.asciidoc +43 -3
- data/lib/logstash/filters/elasticsearch.rb +47 -82
- data/logstash-filter-elasticsearch.gemspec +1 -1
- data/spec/filters/elasticsearch_spec.rb +42 -5
- data/spec/filters/fixtures/request_error.json +25 -0
- data/spec/filters/fixtures/request_x_1.json +5 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 13f03ed46b5acfa84d22de961c7be68dfba316ccf9b4c3930405aa83454008ec
|
4
|
+
data.tar.gz: 32c91902583d770b2f98ec0a65d6b5f24006e11509978d90e19cb14c10828852
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 10989b0b2dc5369c11b82b7d88b6dd4e899ace3dcb2b010051daf6db1fb672ce9762a1faa87d153652b52c659864a163b22512428a1f8a8f5df5c7ae6141a7ff
|
7
|
+
data.tar.gz: 5c4a7e1cf126047839893544073942f0ca48d438d19519b7c7a7c63f299de108dde3c0fdf107b5cc2d9e38f51ec09907af1e77ce7e3004adff4b4bfe26bf2ad6
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 3.3.0
|
2
|
+
- Enhancement : if elasticsearch response contains any shard failure, then `tag_on_failure` tags are added to Logstash event
|
3
|
+
- Enhancement : add support for nested fields
|
4
|
+
- Enhancement : add 'docinfo_fields' option
|
5
|
+
- Enhancement : add 'aggregation_fields' option
|
6
|
+
|
1
7
|
## 3.2.1
|
2
8
|
- Update gemspec summary
|
3
9
|
|
data/LICENSE
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -55,7 +55,7 @@ the "end" event. Finally, using a combination of the "date" filter and the
|
|
55
55
|
}
|
56
56
|
|
57
57
|
ruby {
|
58
|
-
code => "event
|
58
|
+
code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)"
|
59
59
|
}
|
60
60
|
}
|
61
61
|
|
@@ -67,6 +67,7 @@ the "end" event. Finally, using a combination of the "date" filter and the
|
|
67
67
|
elasticsearch {
|
68
68
|
hosts => ["es-server"]
|
69
69
|
query_template => "template.json"
|
70
|
+
fields => { "@timestamp" => "started" }
|
70
71
|
}
|
71
72
|
|
72
73
|
date {
|
@@ -75,7 +76,7 @@ the "end" event. Finally, using a combination of the "date" filter and the
|
|
75
76
|
}
|
76
77
|
|
77
78
|
ruby {
|
78
|
-
code => "event
|
79
|
+
code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600)"
|
79
80
|
}
|
80
81
|
}
|
81
82
|
|
@@ -89,7 +90,7 @@ the "end" event. Finally, using a combination of the "date" filter and the
|
|
89
90
|
"query": "type:start AND operation:%{[opid]}"
|
90
91
|
}
|
91
92
|
},
|
92
|
-
"_source": ["@timestamp"
|
93
|
+
"_source": ["@timestamp"]
|
93
94
|
}
|
94
95
|
|
95
96
|
As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
|
@@ -105,7 +106,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
105
106
|
[cols="<,<,<",options="header",]
|
106
107
|
|=======================================================================
|
107
108
|
|Setting |Input type|Required
|
109
|
+
| <<plugins-{type}s-{plugin}-aggregation_fields>> |<<hash,hash>>|No
|
108
110
|
| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
|
111
|
+
| <<plugins-{type}s-{plugin}-docinfo_fields>> |<<hash,hash>>|No
|
109
112
|
| <<plugins-{type}s-{plugin}-enable_sort>> |<<boolean,boolean>>|No
|
110
113
|
| <<plugins-{type}s-{plugin}-fields>> |<<array,array>>|No
|
111
114
|
| <<plugins-{type}s-{plugin}-hosts>> |<<array,array>>|No
|
@@ -125,6 +128,24 @@ filter plugins.
|
|
125
128
|
|
126
129
|
|
127
130
|
|
131
|
+
[id="plugins-{type}s-{plugin}-aggregation_fields"]
|
132
|
+
===== `aggregation_fields`
|
133
|
+
|
134
|
+
* Value type is <<hash,hash>>
|
135
|
+
* Default value is `{}`
|
136
|
+
|
137
|
+
Hash of aggregation names to copy from elasticsearch response into Logstash event fields
|
138
|
+
|
139
|
+
Example:
|
140
|
+
[source,ruby]
|
141
|
+
filter {
|
142
|
+
elasticsearch {
|
143
|
+
aggregation_fields => {
|
144
|
+
"my_agg_name" => "my_ls_field"
|
145
|
+
}
|
146
|
+
}
|
147
|
+
}
|
148
|
+
|
128
149
|
[id="plugins-{type}s-{plugin}-ca_file"]
|
129
150
|
===== `ca_file`
|
130
151
|
|
@@ -133,6 +154,25 @@ filter plugins.
|
|
133
154
|
|
134
155
|
SSL Certificate Authority file
|
135
156
|
|
157
|
+
[id="plugins-{type}s-{plugin}-docinfo_fields"]
|
158
|
+
===== `docinfo_fields`
|
159
|
+
|
160
|
+
* Value type is <<hash,hash>>
|
161
|
+
* Default value is `{}`
|
162
|
+
|
163
|
+
Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
|
164
|
+
|
165
|
+
Example:
|
166
|
+
[source,ruby]
|
167
|
+
filter {
|
168
|
+
elasticsearch {
|
169
|
+
docinfo_fields => {
|
170
|
+
"_id" => "document_id"
|
171
|
+
"_index" => "document_index"
|
172
|
+
}
|
173
|
+
}
|
174
|
+
}
|
175
|
+
|
136
176
|
[id="plugins-{type}s-{plugin}-enable_sort"]
|
137
177
|
===== `enable_sort`
|
138
178
|
|
@@ -5,82 +5,6 @@ require_relative "elasticsearch/client"
|
|
5
5
|
require "logstash/json"
|
6
6
|
java_import "java.util.concurrent.ConcurrentHashMap"
|
7
7
|
|
8
|
-
# .Compatibility Note
|
9
|
-
# [NOTE]
|
10
|
-
# ================================================================================
|
11
|
-
# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
|
12
|
-
# called `http.content_type.required`. If this option is set to `true`, and you
|
13
|
-
# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch filter
|
14
|
-
# plugin to version 3.1.1 or higher.
|
15
|
-
#
|
16
|
-
# ================================================================================
|
17
|
-
#
|
18
|
-
# Search Elasticsearch for a previous log event and copy some fields from it
|
19
|
-
# into the current event. Below are two complete examples of how this filter might
|
20
|
-
# be used.
|
21
|
-
#
|
22
|
-
# The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
|
23
|
-
# Whenever logstash receives an "end" event, it uses this elasticsearch
|
24
|
-
# filter to find the matching "start" event based on some operation identifier.
|
25
|
-
# Then it copies the `@timestamp` field from the "start" event into a new field on
|
26
|
-
# the "end" event. Finally, using a combination of the "date" filter and the
|
27
|
-
# "ruby" filter, we calculate the time duration in hours between the two events.
|
28
|
-
# [source,ruby]
|
29
|
-
# --------------------------------------------------
|
30
|
-
# if [type] == "end" {
|
31
|
-
# elasticsearch {
|
32
|
-
# hosts => ["es-server"]
|
33
|
-
# query => "type:start AND operation:%{[opid]}"
|
34
|
-
# fields => { "@timestamp" => "started" }
|
35
|
-
# }
|
36
|
-
#
|
37
|
-
# date {
|
38
|
-
# match => ["[started]", "ISO8601"]
|
39
|
-
# target => "[started]"
|
40
|
-
# }
|
41
|
-
#
|
42
|
-
# ruby {
|
43
|
-
# code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600) rescue nil"
|
44
|
-
# }
|
45
|
-
# }
|
46
|
-
#
|
47
|
-
# The example below reproduces the above example but utilises the query_template. This query_template represents a full
|
48
|
-
# Elasticsearch query DSL and supports the standard Logstash field substitution syntax. The example below issues
|
49
|
-
# the same query as the first example but uses the template shown.
|
50
|
-
#
|
51
|
-
# if [type] == "end" {
|
52
|
-
# elasticsearch {
|
53
|
-
# hosts => ["es-server"]
|
54
|
-
# query_template => "template.json"
|
55
|
-
# }
|
56
|
-
#
|
57
|
-
# date {
|
58
|
-
# match => ["started", "ISO8601"]
|
59
|
-
# target => "started"
|
60
|
-
# }
|
61
|
-
#
|
62
|
-
# ruby {
|
63
|
-
# code => "event.set('duration_hrs', (event.get('@timestamp') - event.get('started')) / 3600) rescue nil"
|
64
|
-
# }
|
65
|
-
# }
|
66
|
-
#
|
67
|
-
#
|
68
|
-
#
|
69
|
-
# template.json:
|
70
|
-
#
|
71
|
-
# {
|
72
|
-
# "query": {
|
73
|
-
# "query_string": {
|
74
|
-
# "query": "type:start AND operation:%{[opid]}"
|
75
|
-
# }
|
76
|
-
# },
|
77
|
-
# "_source": ["@timestamp", "started"]
|
78
|
-
# }
|
79
|
-
#
|
80
|
-
# As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
|
81
|
-
# The template will be populated per event prior to being used to query Elasticsearch.
|
82
|
-
#
|
83
|
-
# --------------------------------------------------
|
84
8
|
|
85
9
|
class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
86
10
|
config_name "elasticsearch"
|
@@ -106,6 +30,12 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
106
30
|
# Array of fields to copy from old event (found via elasticsearch) into new event
|
107
31
|
config :fields, :validate => :array, :default => {}
|
108
32
|
|
33
|
+
# Hash of docinfo fields to copy from old event (found via elasticsearch) into new event
|
34
|
+
config :docinfo_fields, :validate => :hash, :default => {}
|
35
|
+
|
36
|
+
# Hash of aggregation names to copy from elasticsearch response into Logstash event fields
|
37
|
+
config :aggregation_fields, :validate => :hash, :default => {}
|
38
|
+
|
109
39
|
# Basic Auth - username
|
110
40
|
config :user, :validate => :string
|
111
41
|
|
@@ -145,7 +75,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
145
75
|
|
146
76
|
def filter(event)
|
147
77
|
begin
|
148
|
-
|
149
78
|
params = {:index => event.sprintf(@index) }
|
150
79
|
|
151
80
|
if @query_dsl
|
@@ -161,15 +90,33 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
161
90
|
@logger.debug("Querying elasticsearch for lookup", :params => params)
|
162
91
|
|
163
92
|
results = get_client.search(params)
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
93
|
+
raise "Elasticsearch query error: #{results["_shards"]["failures"]}" if results["_shards"].include? "failures"
|
94
|
+
|
95
|
+
resultsHits = results["hits"]["hits"]
|
96
|
+
if !resultsHits.nil? && !resultsHits.empty?
|
97
|
+
@fields.each do |old_key, new_key|
|
98
|
+
old_key_path = extract_path(old_key)
|
99
|
+
set = resultsHits.map do |doc|
|
100
|
+
extract_value(doc["_source"], old_key_path)
|
101
|
+
end
|
102
|
+
event.set(new_key, set.count > 1 ? set : set.first)
|
103
|
+
end
|
104
|
+
@docinfo_fields.each do |old_key, new_key|
|
105
|
+
old_key_path = extract_path(old_key)
|
106
|
+
set = resultsHits.map do |doc|
|
107
|
+
extract_value(doc, old_key_path)
|
169
108
|
end
|
170
109
|
event.set(new_key, set.count > 1 ? set : set.first)
|
171
110
|
end
|
172
111
|
end
|
112
|
+
|
113
|
+
resultsAggs = results["aggregations"]
|
114
|
+
if !resultsAggs.nil? && !resultsAggs.empty?
|
115
|
+
@aggregation_fields.each do |agg_name, ls_field|
|
116
|
+
event.set(ls_field, resultsAggs[agg_name])
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
173
120
|
rescue => e
|
174
121
|
@logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e)
|
175
122
|
@tag_on_failure.each{|tag| event.tag(tag)}
|
@@ -194,4 +141,22 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
194
141
|
def get_client
|
195
142
|
@clients_pool.computeIfAbsent(Thread.current, lambda { |x| new_client })
|
196
143
|
end
|
144
|
+
|
145
|
+
# get an array of path elements from a path reference
|
146
|
+
def extract_path(path_reference)
|
147
|
+
return [path_reference] unless path_reference.start_with?('[') && path_reference.end_with?(']')
|
148
|
+
|
149
|
+
path_reference[1...-1].split('][')
|
150
|
+
end
|
151
|
+
|
152
|
+
# given a Hash and an array of path fragments, returns the value at the path
|
153
|
+
# @param source [Hash{String=>Object}]
|
154
|
+
# @param path [Array{String}]
|
155
|
+
# @return [Object]
|
156
|
+
def extract_value(source, path)
|
157
|
+
path.reduce(source) do |memo, old_key_fragment|
|
158
|
+
break unless memo.include?(old_key_fragment)
|
159
|
+
memo[old_key_fragment]
|
160
|
+
end
|
161
|
+
end
|
197
162
|
end #class LogStash::Filters::Elasticsearch
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-elasticsearch'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.3.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Copies fields from previous log events in Elasticsearch to current events "
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -20,7 +20,9 @@ describe LogStash::Filters::Elasticsearch do
|
|
20
20
|
{
|
21
21
|
"hosts" => ["localhost:9200"],
|
22
22
|
"query" => "response: 404",
|
23
|
-
"fields" =>
|
23
|
+
"fields" => { "response" => "code" },
|
24
|
+
"docinfo_fields" => { "_index" => "es_index" },
|
25
|
+
"aggregation_fields" => { "bytes_avg" => "bytes_avg_ls_field" }
|
24
26
|
}
|
25
27
|
end
|
26
28
|
let(:plugin) { described_class.new(config) }
|
@@ -60,6 +62,8 @@ describe LogStash::Filters::Elasticsearch do
|
|
60
62
|
it "should enhance the current event with new data" do
|
61
63
|
plugin.filter(event)
|
62
64
|
expect(event.get("code")).to eq(404)
|
65
|
+
expect(event.get("es_index")).to eq("logstash-2014.08.26")
|
66
|
+
expect(event.get("bytes_avg_ls_field")["value"]).to eq(294)
|
63
67
|
end
|
64
68
|
|
65
69
|
it "should receive all necessary params to perform the search" do
|
@@ -74,7 +78,7 @@ describe LogStash::Filters::Elasticsearch do
|
|
74
78
|
"index" => "foo*",
|
75
79
|
"hosts" => ["localhost:9200"],
|
76
80
|
"query" => "response: 404",
|
77
|
-
"fields" =>
|
81
|
+
"fields" => { "response" => "code" }
|
78
82
|
}
|
79
83
|
end
|
80
84
|
|
@@ -90,7 +94,7 @@ describe LogStash::Filters::Elasticsearch do
|
|
90
94
|
{
|
91
95
|
"hosts" => ["localhost:9200"],
|
92
96
|
"query" => "response: 404",
|
93
|
-
"fields" =>
|
97
|
+
"fields" => { "response" => "code" },
|
94
98
|
"result_size" => 10
|
95
99
|
}
|
96
100
|
end
|
@@ -125,7 +129,7 @@ describe LogStash::Filters::Elasticsearch do
|
|
125
129
|
{
|
126
130
|
"hosts" => ["localhost:9200"],
|
127
131
|
"query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
|
128
|
-
"fields" =>
|
132
|
+
"fields" => { "response" => "code" },
|
129
133
|
"result_size" => 1
|
130
134
|
}
|
131
135
|
end
|
@@ -154,7 +158,7 @@ describe LogStash::Filters::Elasticsearch do
|
|
154
158
|
"index" => "foo_%{subst_field}*",
|
155
159
|
"hosts" => ["localhost:9200"],
|
156
160
|
"query" => "response: 404",
|
157
|
-
"fields" =>
|
161
|
+
"fields" => { "response" => "code" }
|
158
162
|
}
|
159
163
|
end
|
160
164
|
|
@@ -162,6 +166,39 @@ describe LogStash::Filters::Elasticsearch do
|
|
162
166
|
expect(client).to receive(:search).with({:q => "response: 404", :size => 1, :index => "foo_subst_value*", :sort => "@timestamp:desc"})
|
163
167
|
plugin.filter(event)
|
164
168
|
end
|
169
|
+
end
|
170
|
+
|
171
|
+
context "if query result errored but no exception is thrown" do
|
172
|
+
let(:response) do
|
173
|
+
LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_error.json")))
|
174
|
+
end
|
175
|
+
|
176
|
+
before(:each) do
|
177
|
+
allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
|
178
|
+
allow(client).to receive(:search).and_return(response)
|
179
|
+
plugin.register
|
180
|
+
end
|
181
|
+
|
182
|
+
it "tag the event as something happened, but still deliver it" do
|
183
|
+
expect(plugin.logger).to receive(:warn)
|
184
|
+
plugin.filter(event)
|
185
|
+
expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
context "if query is on nested field" do
|
190
|
+
let(:config) do
|
191
|
+
{
|
192
|
+
"hosts" => ["localhost:9200"],
|
193
|
+
"query" => "response: 404",
|
194
|
+
"fields" => [ ["[geoip][ip]", "ip_address"] ]
|
195
|
+
}
|
196
|
+
end
|
197
|
+
|
198
|
+
it "should enhance the current event with new data" do
|
199
|
+
plugin.filter(event)
|
200
|
+
expect(event.get("ip_address")).to eq("66.249.73.185")
|
201
|
+
end
|
165
202
|
|
166
203
|
end
|
167
204
|
|
@@ -0,0 +1,25 @@
|
|
1
|
+
{
|
2
|
+
"took": 16,
|
3
|
+
"timed_out": false,
|
4
|
+
"_shards": {
|
5
|
+
"total": 155,
|
6
|
+
"successful": 100,
|
7
|
+
"failed": 55,
|
8
|
+
"failures": [
|
9
|
+
{
|
10
|
+
"shard": 0,
|
11
|
+
"index": "logstash-2014.08.26",
|
12
|
+
"node": "YI1MT0H-Q469pFgAVTXI2g",
|
13
|
+
"reason": {
|
14
|
+
"type": "search_parse_exception",
|
15
|
+
"reason": "No mapping found for [@timestamp] in order to sort on"
|
16
|
+
}
|
17
|
+
}
|
18
|
+
]
|
19
|
+
},
|
20
|
+
"hits": {
|
21
|
+
"total": 0,
|
22
|
+
"max_score": null,
|
23
|
+
"hits": []
|
24
|
+
}
|
25
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-01-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,6 +84,7 @@ files:
|
|
84
84
|
- logstash-filter-elasticsearch.gemspec
|
85
85
|
- spec/filters/elasticsearch_spec.rb
|
86
86
|
- spec/filters/fixtures/query_template.json
|
87
|
+
- spec/filters/fixtures/request_error.json
|
87
88
|
- spec/filters/fixtures/request_x_1.json
|
88
89
|
- spec/filters/fixtures/request_x_10.json
|
89
90
|
- spec/filters/integration/elasticsearch_spec.rb
|
@@ -109,13 +110,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
109
110
|
version: '0'
|
110
111
|
requirements: []
|
111
112
|
rubyforge_project:
|
112
|
-
rubygems_version: 2.6.
|
113
|
+
rubygems_version: 2.6.13
|
113
114
|
signing_key:
|
114
115
|
specification_version: 4
|
115
116
|
summary: Copies fields from previous log events in Elasticsearch to current events
|
116
117
|
test_files:
|
117
118
|
- spec/filters/elasticsearch_spec.rb
|
118
119
|
- spec/filters/fixtures/query_template.json
|
120
|
+
- spec/filters/fixtures/request_error.json
|
119
121
|
- spec/filters/fixtures/request_x_1.json
|
120
122
|
- spec/filters/fixtures/request_x_10.json
|
121
123
|
- spec/filters/integration/elasticsearch_spec.rb
|