logstash-filter-elasticsearch 3.1.6 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 364c2ed330ed87b3914f8a7644460ed48522b5db
|
4
|
+
data.tar.gz: a77e91ea3a1071c78742fc9ce972f0d5ee9791f0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c7df570123dc569cdffd64f310b154f735efe97306b93bd7c76ce8cc43a92f67b9558b16d27d99f653e454473dad2ddf3e95703ea16229625b043d8b5e723a4d
|
7
|
+
data.tar.gz: f64a717402ab19107450d6d800d3f5abb0ee32075b9c9afccdb61d8ba891b2f52f25a23c69117ecc0f816c42b42b8f9caa8e7e59b7fd44df5d1930cb58e1d04d
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## 3.2.0
|
2
|
+
- `index` setting now supports field formatting, such as `index => "%{myindex}"` (Boris Gorbylev)
|
3
|
+
|
4
|
+
## 3.1.8
|
5
|
+
- Fix a thread safety issue when using this filter with multiple workers on heavy load, we now create an elasticsearch client for every LogStash worker. #76
|
6
|
+
|
1
7
|
## 3.1.6
|
2
8
|
- Fix some documentation issues
|
3
9
|
|
data/docs/index.asciidoc
CHANGED
@@ -163,7 +163,8 @@ List of elasticsearch hosts to use for querying.
|
|
163
163
|
* Value type is <<string,string>>
|
164
164
|
* Default value is `""`
|
165
165
|
|
166
|
-
Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
|
166
|
+
Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
|
167
|
+
Field substitution (e.g. `index-name-%{date_field}`) is available
|
167
168
|
|
168
169
|
[id="plugins-{type}s-{plugin}-password"]
|
169
170
|
===== `password`
|
@@ -3,6 +3,7 @@ require "logstash/filters/base"
|
|
3
3
|
require "logstash/namespace"
|
4
4
|
require_relative "elasticsearch/client"
|
5
5
|
require "logstash/json"
|
6
|
+
java_import "java.util.concurrent.ConcurrentHashMap"
|
6
7
|
|
7
8
|
# .Compatibility Note
|
8
9
|
# [NOTE]
|
@@ -87,7 +88,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
87
88
|
# List of elasticsearch hosts to use for querying.
|
88
89
|
config :hosts, :validate => :array, :default => [ "localhost:9200" ]
|
89
90
|
|
90
|
-
# Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
|
91
|
+
# Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices.
|
92
|
+
# Field substitution (e.g. `index-name-%{date_field}`) is available
|
91
93
|
config :index, :validate => :string, :default => ""
|
92
94
|
|
93
95
|
# Elasticsearch query string. Read the Elasticsearch query string documentation.
|
@@ -125,14 +127,10 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
125
127
|
# Tags the event on failure to look up geo information. This can be used in later analysis.
|
126
128
|
config :tag_on_failure, :validate => :array, :default => ["_elasticsearch_lookup_failure"]
|
127
129
|
|
130
|
+
attr_reader :clients_pool
|
131
|
+
|
128
132
|
def register
|
129
|
-
|
130
|
-
:ssl => @ssl,
|
131
|
-
:hosts => @hosts,
|
132
|
-
:ca_file => @ca_file,
|
133
|
-
:logger => @logger
|
134
|
-
}
|
135
|
-
@client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options)
|
133
|
+
@clients_pool = java.util.concurrent.ConcurrentHashMap.new
|
136
134
|
|
137
135
|
#Load query if it exists
|
138
136
|
if @query_template
|
@@ -148,7 +146,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
148
146
|
def filter(event)
|
149
147
|
begin
|
150
148
|
|
151
|
-
params = {:index => @index }
|
149
|
+
params = {:index => event.sprintf(@index) }
|
152
150
|
|
153
151
|
if @query_dsl
|
154
152
|
query = LogStash::Json.load(event.sprintf(@query_dsl))
|
@@ -162,7 +160,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
162
160
|
|
163
161
|
@logger.debug("Querying elasticsearch for lookup", :params => params)
|
164
162
|
|
165
|
-
results =
|
163
|
+
results = get_client.search(params)
|
166
164
|
@fields.each do |old_key, new_key|
|
167
165
|
if !results['hits']['hits'].empty?
|
168
166
|
set = []
|
@@ -178,4 +176,22 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
178
176
|
end
|
179
177
|
filter_matched(event)
|
180
178
|
end # def filter
|
179
|
+
|
180
|
+
private
|
181
|
+
def client_options
|
182
|
+
{
|
183
|
+
:ssl => @ssl,
|
184
|
+
:hosts => @hosts,
|
185
|
+
:ca_file => @ca_file,
|
186
|
+
:logger => @logger
|
187
|
+
}
|
188
|
+
end
|
189
|
+
|
190
|
+
def new_client
|
191
|
+
LogStash::Filters::ElasticsearchClient.new(@user, @password, client_options)
|
192
|
+
end
|
193
|
+
|
194
|
+
def get_client
|
195
|
+
@clients_pool.computeIfAbsent(Thread.current, lambda { |x| new_client })
|
196
|
+
end
|
181
197
|
end #class LogStash::Filters::Elasticsearch
|
@@ -24,7 +24,7 @@ module LogStash
|
|
24
24
|
# set ca_file even if ssl isn't on, since the host can be an https url
|
25
25
|
transport_options[:ssl] = { ca_file: options[:ca_file] } if options[:ca_file]
|
26
26
|
|
27
|
-
@logger.info("New ElasticSearch filter", :hosts => hosts)
|
27
|
+
@logger.info("New ElasticSearch filter client", :hosts => hosts)
|
28
28
|
@client = ::Elasticsearch::Client.new(hosts: hosts, transport_options: transport_options)
|
29
29
|
end
|
30
30
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-elasticsearch'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.2.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -38,6 +38,25 @@ describe LogStash::Filters::Elasticsearch do
|
|
38
38
|
plugin.register
|
39
39
|
end
|
40
40
|
|
41
|
+
after(:each) do
|
42
|
+
Thread.current[:filter_elasticsearch_client] = nil
|
43
|
+
end
|
44
|
+
|
45
|
+
# Since the Elasticsearch Ruby client is not thread safe
|
46
|
+
# and under high load we can get error with the connection pool
|
47
|
+
# we have decided to create a new instance per worker thread which
|
48
|
+
# will be lazy created on the first call to `#filter`
|
49
|
+
#
|
50
|
+
# I am adding a simple test case for future changes
|
51
|
+
it "uses a different connection object per thread wait" do
|
52
|
+
expect(plugin.clients_pool.size).to eq(0)
|
53
|
+
|
54
|
+
Thread.new { plugin.filter(event) }.join
|
55
|
+
Thread.new { plugin.filter(event) }.join
|
56
|
+
|
57
|
+
expect(plugin.clients_pool.size).to eq(2)
|
58
|
+
end
|
59
|
+
|
41
60
|
it "should enhance the current event with new data" do
|
42
61
|
plugin.filter(event)
|
43
62
|
expect(event.get("code")).to eq(404)
|
@@ -122,6 +141,30 @@ describe LogStash::Filters::Elasticsearch do
|
|
122
141
|
|
123
142
|
end
|
124
143
|
|
144
|
+
context "testing a simple index substitution" do
|
145
|
+
let(:event) {
|
146
|
+
LogStash::Event.new(
|
147
|
+
{
|
148
|
+
"subst_field" => "subst_value"
|
149
|
+
}
|
150
|
+
)
|
151
|
+
}
|
152
|
+
let(:config) do
|
153
|
+
{
|
154
|
+
"index" => "foo_%{subst_field}*",
|
155
|
+
"hosts" => ["localhost:9200"],
|
156
|
+
"query" => "response: 404",
|
157
|
+
"fields" => [["response", "code"]]
|
158
|
+
}
|
159
|
+
end
|
160
|
+
|
161
|
+
it "should receive substituted index name" do
|
162
|
+
expect(client).to receive(:search).with({:q => "response: 404", :size => 1, :index => "foo_subst_value*", :sort => "@timestamp:desc"})
|
163
|
+
plugin.filter(event)
|
164
|
+
end
|
165
|
+
|
166
|
+
end
|
167
|
+
|
125
168
|
end
|
126
169
|
|
127
170
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-09-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|