logstash-filter-elasticsearch 3.0.2 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/lib/logstash/filters/elasticsearch.rb +77 -11
- data/lib/logstash/filters/elasticsearch/client.rb +3 -2
- data/logstash-filter-elasticsearch.gemspec +1 -1
- data/spec/filters/elasticsearch_spec.rb +22 -0
- data/spec/filters/fixtures/query_template.json +7 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ff652465fd4904e101402ef5c0085fa249859e6
|
4
|
+
data.tar.gz: 594e66f78a3c9676391b9d58e2e3d1eb6fb5cf8a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1a48fb3287a46b475ce29dd03e2982c1a246b031f4f45fb951606f5819846b76bd6a28c1392a0c2e836f29eef086105cce715849eee35d634d546769c2c04b25
|
7
|
+
data.tar.gz: f2ee559904b31e25b7c0b8f6ed212f096babe29d5b11637a88f222522a93566d98c71d947d735b2bab96fc6deefa2a799f5540ef40c9790863b6073348987eeb
|
data/CHANGELOG.md
CHANGED
@@ -2,21 +2,25 @@
|
|
2
2
|
require "logstash/filters/base"
|
3
3
|
require "logstash/namespace"
|
4
4
|
require_relative "elasticsearch/client"
|
5
|
+
require "logstash/json"
|
5
6
|
|
6
|
-
|
7
|
-
#
|
8
|
-
#
|
9
|
-
#
|
7
|
+
# Search Elasticsearch for a previous log event and copy some fields from it
|
8
|
+
# into the current event. Below are two complete examples of how this filter might
|
9
|
+
# be used.
|
10
|
+
#
|
11
|
+
# The first example uses the legacy 'query' parameter where the user is limited to an Elasticsearch query_string.
|
12
|
+
# Whenever logstash receives an "end" event, it uses this elasticsearch
|
10
13
|
# filter to find the matching "start" event based on some operation identifier.
|
11
14
|
# Then it copies the `@timestamp` field from the "start" event into a new field on
|
12
15
|
# the "end" event. Finally, using a combination of the "date" filter and the
|
13
16
|
# "ruby" filter, we calculate the time duration in hours between the two events.
|
14
17
|
# [source,ruby]
|
18
|
+
# --------------------------------------------------
|
15
19
|
# if [type] == "end" {
|
16
20
|
# elasticsearch {
|
17
21
|
# hosts => ["es-server"]
|
18
22
|
# query => "type:start AND operation:%{[opid]}"
|
19
|
-
# fields =>
|
23
|
+
# fields => { "@timestamp" => "started" }
|
20
24
|
# }
|
21
25
|
#
|
22
26
|
# date {
|
@@ -29,6 +33,44 @@ require_relative "elasticsearch/client"
|
|
29
33
|
# }
|
30
34
|
# }
|
31
35
|
#
|
36
|
+
# The example below reproduces the above example but utilises the query_template. This query_template represents a full
|
37
|
+
# Elasticsearch query DSL and supports the standard Logstash field substitution syntax. The example below issues
|
38
|
+
# the same query as the first example but uses the template shown.
|
39
|
+
#
|
40
|
+
# if [type] == "end" {
|
41
|
+
# elasticsearch {
|
42
|
+
# hosts => ["es-server"]
|
43
|
+
# query_template => "template.json"
|
44
|
+
# }
|
45
|
+
#
|
46
|
+
# date {
|
47
|
+
# match => ["[started]", "ISO8601"]
|
48
|
+
# target => "[started]"
|
49
|
+
# }
|
50
|
+
#
|
51
|
+
# ruby {
|
52
|
+
# code => "event['duration_hrs'] = (event['@timestamp'] - event['started']) / 3600 rescue nil"
|
53
|
+
# }
|
54
|
+
# }
|
55
|
+
#
|
56
|
+
#
|
57
|
+
#
|
58
|
+
# template.json:
|
59
|
+
#
|
60
|
+
# {
|
61
|
+
# "query": {
|
62
|
+
# "query_string": {
|
63
|
+
# "query": "type:start AND operation:%{[opid]}"
|
64
|
+
# }
|
65
|
+
# },
|
66
|
+
# "_source": ["@timestamp", "started"]
|
67
|
+
# }
|
68
|
+
#
|
69
|
+
# As illustrated above, through the use of 'opid', fields from the Logstash events can be referenced within the template.
|
70
|
+
# The template will be populated per event prior to being used to query Elasticsearch.
|
71
|
+
#
|
72
|
+
# --------------------------------------------------
|
73
|
+
|
32
74
|
class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
33
75
|
config_name "elasticsearch"
|
34
76
|
|
@@ -38,10 +80,14 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
38
80
|
# Comma-delimited list of index names to search; use `_all` or empty string to perform the operation on all indices
|
39
81
|
config :index, :validate => :string, :default => ""
|
40
82
|
|
41
|
-
# Elasticsearch query string. Read the Elasticsearch query string documentation
|
83
|
+
# Elasticsearch query string. Read the Elasticsearch query string documentation.
|
42
84
|
# for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/master/query-dsl-query-string-query.html#query-string-syntax
|
43
85
|
config :query, :validate => :string
|
44
86
|
|
87
|
+
# File path to elasticsearch query in DSL format. Read the Elasticsearch query documentation
|
88
|
+
# for more info at: https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
|
89
|
+
config :query_template, :validate => :string
|
90
|
+
|
45
91
|
# Comma-delimited list of `<field>:<direction>` pairs that define the sort order
|
46
92
|
config :sort, :validate => :string, :default => "@timestamp:desc"
|
47
93
|
|
@@ -77,16 +123,36 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
77
123
|
:logger => @logger
|
78
124
|
}
|
79
125
|
@client = LogStash::Filters::ElasticsearchClient.new(@user, @password, options)
|
126
|
+
|
127
|
+
#Load query if it exists
|
128
|
+
if @query_template
|
129
|
+
if File.zero?(@query_template)
|
130
|
+
raise "template is empty"
|
131
|
+
end
|
132
|
+
file = File.open(@query_template, "rb")
|
133
|
+
@query_dsl = file.read
|
134
|
+
end
|
135
|
+
|
80
136
|
end # def register
|
81
137
|
|
82
138
|
def filter(event)
|
83
139
|
begin
|
84
|
-
query_str = event.sprintf(@query)
|
85
140
|
|
86
|
-
params = {
|
87
|
-
params[:sort] = @sort if @enable_sort
|
88
|
-
results = @client.search(params)
|
141
|
+
params = {:index => @index }
|
89
142
|
|
143
|
+
if @query_dsl
|
144
|
+
query = LogStash::Json.load(event.sprintf(@query_dsl))
|
145
|
+
params[:body] = query
|
146
|
+
else
|
147
|
+
query = event.sprintf(@query)
|
148
|
+
params[:q] = query
|
149
|
+
params[:size] = result_size
|
150
|
+
params[:sort] = @sort if @enable_sort
|
151
|
+
end
|
152
|
+
|
153
|
+
@logger.info("Querying elasticsearch for lookup", :params => params)
|
154
|
+
|
155
|
+
results = @client.search(params)
|
90
156
|
@fields.each do |old_key, new_key|
|
91
157
|
if !results['hits']['hits'].empty?
|
92
158
|
set = []
|
@@ -97,7 +163,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
97
163
|
end
|
98
164
|
end
|
99
165
|
rescue => e
|
100
|
-
@logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query =>
|
166
|
+
@logger.warn("Failed to query elasticsearch for previous event", :index => @index, :query => query, :event => event, :error => e)
|
101
167
|
@tag_on_failure.each{|tag| event.tag(tag)}
|
102
168
|
end
|
103
169
|
filter_matched(event)
|
@@ -2,6 +2,7 @@
|
|
2
2
|
require "elasticsearch"
|
3
3
|
require "base64"
|
4
4
|
|
5
|
+
|
5
6
|
module LogStash
|
6
7
|
module Filters
|
7
8
|
class ElasticsearchClient
|
@@ -9,7 +10,7 @@ module LogStash
|
|
9
10
|
attr_reader :client
|
10
11
|
|
11
12
|
def initialize(user, password, options={})
|
12
|
-
ssl = options.fetch(:
|
13
|
+
ssl = options.fetch(:ssl, false)
|
13
14
|
hosts = options[:hosts]
|
14
15
|
@logger = options[:logger]
|
15
16
|
|
@@ -19,7 +20,7 @@ module LogStash
|
|
19
20
|
transport_options[:headers] = { Authorization: "Basic #{token}" }
|
20
21
|
end
|
21
22
|
|
22
|
-
|
23
|
+
hosts.map! {|h| { host: h, scheme: 'https' } } if ssl
|
23
24
|
transport_options[:ssl] = { ca_file: options[:ca_file] } if ssl && options[:ca_file]
|
24
25
|
|
25
26
|
@logger.info("New ElasticSearch filter", :hosts => hosts)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-elasticsearch'
|
4
|
-
s.version = '3.0
|
4
|
+
s.version = '3.1.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Search elasticsearch for a previous log event and copy some fields from it into the current event"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -100,6 +100,28 @@ describe LogStash::Filters::Elasticsearch do
|
|
100
100
|
expect(event.to_hash["tags"]).to include("_elasticsearch_lookup_failure")
|
101
101
|
end
|
102
102
|
end
|
103
|
+
|
104
|
+
context "testing a simple query template" do
|
105
|
+
let(:config) do
|
106
|
+
{
|
107
|
+
"hosts" => ["localhost:9200"],
|
108
|
+
"query_template" => File.join(File.dirname(__FILE__), "fixtures", "query_template.json"),
|
109
|
+
"fields" => [ ["response", "code"] ],
|
110
|
+
"result_size" => 1
|
111
|
+
}
|
112
|
+
end
|
113
|
+
|
114
|
+
let(:response) do
|
115
|
+
LogStash::Json.load(File.read(File.join(File.dirname(__FILE__), "fixtures", "request_x_1.json")))
|
116
|
+
end
|
117
|
+
|
118
|
+
it "should enhance the current event with new data" do
|
119
|
+
plugin.filter(event)
|
120
|
+
expect(event.get("code")).to eq(404)
|
121
|
+
end
|
122
|
+
|
123
|
+
end
|
124
|
+
|
103
125
|
end
|
104
126
|
|
105
127
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-12-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -74,6 +74,7 @@ files:
|
|
74
74
|
- lib/logstash/filters/elasticsearch/client.rb
|
75
75
|
- logstash-filter-elasticsearch.gemspec
|
76
76
|
- spec/filters/elasticsearch_spec.rb
|
77
|
+
- spec/filters/fixtures/query_template.json
|
77
78
|
- spec/filters/fixtures/request_x_1.json
|
78
79
|
- spec/filters/fixtures/request_x_10.json
|
79
80
|
- spec/filters/integration/elasticsearch_spec.rb
|
@@ -99,12 +100,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
99
100
|
version: '0'
|
100
101
|
requirements: []
|
101
102
|
rubyforge_project:
|
102
|
-
rubygems_version: 2.
|
103
|
+
rubygems_version: 2.4.8
|
103
104
|
signing_key:
|
104
105
|
specification_version: 4
|
105
106
|
summary: Search elasticsearch for a previous log event and copy some fields from it into the current event
|
106
107
|
test_files:
|
107
108
|
- spec/filters/elasticsearch_spec.rb
|
109
|
+
- spec/filters/fixtures/query_template.json
|
108
110
|
- spec/filters/fixtures/request_x_1.json
|
109
111
|
- spec/filters/fixtures/request_x_10.json
|
110
112
|
- spec/filters/integration/elasticsearch_spec.rb
|