logstash-filter-lookup 1.2.1 → 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/README.md +2 -3
- data/lib/logstash/filters/lookup.rb +97 -82
- data/logstash-filter-lookup.gemspec +9 -8
- data/spec/filters/lookup_spec.rb +257 -79
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 175c91bd50e708f7d89b7db370bac124f3770d60
|
4
|
+
data.tar.gz: fb814d76271fa06046fab307df9ad0fc169d684d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9385b8dd385a3fe54b2908b81e7cab60179df6d5b9e26e1ecdab131b690baf56d94847ed7e8bc8ceac45889b6d1c6382b0e7d67ab310d41506ea54c7650a805a
|
7
|
+
data.tar.gz: 063b615c61acc2d109b97496b6f094b90ff4eb037a24517939902ad0d6b75297ca0ed338965d64b83d918d80f111d8fd15e9224c60f0402c6b570e2d24f474b1
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -1,8 +1,7 @@
|
|
1
|
-
#
|
1
|
+
# logstash-filter-lookup
|
2
2
|
|
3
|
-
|
3
|
+
New lookup plugin for dynamic lookup enrichment from multiple types of sources. There are a few different filters out there today that conduct different types of lookups, which may make sense being encapsulated within one lookup filter.
|
4
4
|
|
5
|
-
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
6
5
|
|
7
6
|
## Example
|
8
7
|
### 1. Web Service Example
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'logstash/filters/base'
|
3
3
|
require 'logstash/namespace'
|
4
|
-
require 'open-uri'
|
5
|
-
require 'digest/sha1'
|
6
4
|
require 'json'
|
7
5
|
require 'csv'
|
6
|
+
require 'rest_client'
|
7
|
+
|
8
8
|
|
9
9
|
# A general search and replace tool which uses a Web service with a YAML, CSV or JSON response to determine replacement values.
|
10
10
|
#
|
@@ -14,10 +14,7 @@ require 'csv'
|
|
14
14
|
# matches the EXACT contents of a map entry key, the field's value will be substituted
|
15
15
|
# with the matched key's value from the map.
|
16
16
|
#
|
17
|
-
#
|
18
|
-
# maching event field (in-place). However, by using the `destination`
|
19
|
-
# configuration item, you may also specify a target event field to
|
20
|
-
# populate with the new mapd value.
|
17
|
+
# The lookup filter will put the contents of the maching event fields into field result.
|
21
18
|
|
22
19
|
class LogStash::Filters::LookUp < LogStash::Filters::Base
|
23
20
|
config_name "lookup"
|
@@ -25,56 +22,59 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base
|
|
25
22
|
# match by the map filter (e.g. `message`, `host`, `response_code`).
|
26
23
|
#
|
27
24
|
# If this field is an array, only the first value will be used.
|
28
|
-
config :
|
25
|
+
config :fields, :validate => :array, :required => true
|
29
26
|
|
30
|
-
#
|
31
|
-
#
|
32
|
-
# value with the new mapping value.
|
27
|
+
# By default false, if false, then all the result will be stored into a field called lookup_result
|
28
|
+
# If true, the same field used to lookup, will be replaced.
|
33
29
|
config :override, :validate => :boolean, :default => false
|
34
30
|
|
31
|
+
# Can be webservice or file
|
32
|
+
config :type, :validate => :string, :default => 'webservice'
|
33
|
+
|
34
|
+
# Path of a file
|
35
|
+
config :path, :validate => :string
|
36
|
+
|
37
|
+
config :format, :validate => :string, :default => 'yml'
|
38
|
+
|
39
|
+
config :headers, :validate => :hash, :default => { }
|
40
|
+
config :params, :validate => :hash, :default => { }
|
41
|
+
|
42
|
+
#default field where store results if override option is false
|
43
|
+
config :result_key, :validate => :string, :default => "lookup_result"
|
44
|
+
|
35
45
|
# The full URI path of a Web service who generates an JSON, yml or CSV format response.
|
36
46
|
# requires to append as suffix the format type. Ex: http://localhost:8080/geoPoints?type=json
|
37
47
|
# http://localhost:8080/geoPoints/json
|
38
48
|
# http://localhost:8080/geoPoints/csv
|
39
49
|
# If no suffix matches, defaults to YAML
|
40
|
-
config :
|
50
|
+
config :url, :validate => :string
|
51
|
+
|
52
|
+
#HTTP method
|
53
|
+
config :method, :validate => :string, :default => "get"
|
54
|
+
|
55
|
+
|
41
56
|
|
42
57
|
# When using a map file or url, this setting will indicate how frequently
|
43
58
|
# (in seconds) logstash will check the YAML file or url for updates.
|
44
59
|
config :refresh_interval, :validate => :number, :default => 300
|
45
60
|
|
46
|
-
# The destination field you wish to populate with the mapd code. The default
|
47
|
-
# is a field named `mapping`. Set this to the same value as source if you want
|
48
|
-
# to do a substitution, in this case filter will allways succeed. This will clobber
|
49
|
-
# the old value of the source field!
|
50
|
-
config :destination, :validate => :string, :default => "mapping"
|
51
|
-
|
52
61
|
# In case no mapping occurs in the event (no matches), this will add a default
|
53
62
|
# mapping string, which will always populate `field`, if the match failed.
|
54
|
-
|
55
|
-
# For example, if we have configured `fallback => "no match"`, using this map:
|
56
|
-
# [source,ruby]
|
57
|
-
# foo: bar
|
58
|
-
#
|
59
|
-
# Then, if logstash received an event with the field `foo` set to `bar`, the destination
|
60
|
-
# field would be set to `bar`. However, if logstash received an event with `foo` set to `nope`,
|
61
|
-
# then the destination field would still be populated, but with the value of `no match`.
|
62
|
-
# This configuration can be dynamic and include parts of the event using the `%{field}` syntax.
|
63
|
-
config :fallback, :validate => :string
|
63
|
+
config :default_values, :validate => :array, :default => {}
|
64
64
|
|
65
65
|
def get_map
|
66
66
|
@my_map
|
67
67
|
end
|
68
68
|
|
69
|
-
def
|
70
|
-
|
69
|
+
def fill_map(data)
|
70
|
+
get_map.merge!(data)
|
71
71
|
end
|
72
72
|
|
73
73
|
public
|
74
74
|
def register
|
75
75
|
@my_map = {}
|
76
76
|
@next_refresh = Time.now + @refresh_interval
|
77
|
-
|
77
|
+
load(true)
|
78
78
|
@logger.debug? and @logger.debug("#{self.class.name}: map - ", :map => get_map)
|
79
79
|
type = 'Exact'
|
80
80
|
@logger.debug? and @logger.debug("#{self.class.name}: map mapping method - "+type)
|
@@ -83,101 +83,116 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base
|
|
83
83
|
# def register
|
84
84
|
|
85
85
|
def json_loader(data)
|
86
|
-
|
86
|
+
fill_map(JSON.parse(data))
|
87
87
|
end
|
88
88
|
|
89
89
|
def csv_loader(data)
|
90
|
-
data = CSV.
|
90
|
+
data = CSV.parse(data).inject(Hash.new) do |acc, v|
|
91
91
|
acc[v[0]] = v[1]
|
92
92
|
acc
|
93
93
|
end
|
94
|
-
|
94
|
+
fill_map(data)
|
95
95
|
end
|
96
96
|
|
97
97
|
def yml_loader(data)
|
98
|
-
|
98
|
+
fill_map(YAML.load(data))
|
99
99
|
end
|
100
100
|
|
101
|
-
def
|
101
|
+
def load_data(registering, data)
|
102
102
|
begin
|
103
|
-
if
|
103
|
+
if @format.eql?('json')
|
104
104
|
return json_loader(data)
|
105
|
-
elsif
|
105
|
+
elsif @format.eql?('csv')
|
106
106
|
return csv_loader(data)
|
107
107
|
end
|
108
108
|
yml_loader(data)
|
109
109
|
rescue Exception => _
|
110
110
|
if registering
|
111
|
-
raise "#{self.class.name}: Bad Syntax in
|
111
|
+
raise "#{self.class.name}: Bad Syntax in data #{data}"
|
112
112
|
else
|
113
|
-
@logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path =>
|
113
|
+
@logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => data)
|
114
114
|
end
|
115
115
|
end
|
116
116
|
end
|
117
117
|
|
118
|
-
def
|
119
|
-
if path.end_with?('json')
|
120
|
-
return '.json'
|
121
|
-
elsif path.end_with?('csv')
|
122
|
-
return '.csv'
|
123
|
-
end
|
124
|
-
'.yml'
|
125
|
-
end
|
126
|
-
|
127
|
-
public
|
128
|
-
def download_ws(path, registering=false)
|
129
|
-
extension = get_extension(path)
|
130
|
-
temp_extension = '_temp'+extension;
|
131
|
-
file_name = Digest::SHA1.hexdigest path
|
118
|
+
def load(registering=false)
|
132
119
|
begin
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
120
|
+
if @type=='webservice'
|
121
|
+
route = @url
|
122
|
+
data = get_webservice_content(route)
|
123
|
+
elsif @type=='file'
|
124
|
+
route = @path
|
125
|
+
data = get_file_content(route)
|
137
126
|
end
|
127
|
+
load_data(registering, data)
|
138
128
|
rescue Exception => _
|
139
129
|
if registering
|
140
|
-
raise "#{self.class.name}: Failed to initialize with #{
|
130
|
+
raise "#{self.class.name}: Failed to initialize with type #{type} and route #{route}"
|
141
131
|
end
|
142
|
-
@logger.warn("#{self.class.name}: Something happened with
|
132
|
+
@logger.warn("#{self.class.name}: Something happened with load. Continuing with old map", :type => @type, :route => route)
|
143
133
|
end
|
134
|
+
end
|
144
135
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
136
|
+
# def load
|
137
|
+
|
138
|
+
def get_file_content(file)
|
139
|
+
data = ''
|
140
|
+
File.open(file, 'r') do |read_file|
|
141
|
+
data +=read_file.read
|
142
|
+
end
|
143
|
+
data
|
144
|
+
end
|
145
|
+
|
146
|
+
# def get_file_content
|
147
|
+
|
148
|
+
def get_webservice_content(path)
|
149
|
+
case @method
|
150
|
+
when "get"
|
151
|
+
data = RestClient.get sprint(path), sprint(@headers)
|
152
|
+
when "post"
|
153
|
+
data = RestClient.post sprint(path), sprint(@params), sprint(@headers)
|
150
154
|
end
|
155
|
+
data
|
151
156
|
end
|
152
157
|
|
153
|
-
# def
|
158
|
+
# def get_webservice_content
|
159
|
+
|
160
|
+
def sprint(hash)
|
161
|
+
hash
|
162
|
+
end
|
154
163
|
|
155
164
|
public
|
156
165
|
def filter(event)
|
157
166
|
if @next_refresh < Time.now
|
158
|
-
|
167
|
+
load()
|
159
168
|
@next_refresh = Time.now + @refresh_interval
|
160
169
|
@logger.info('downloading and refreshing map file')
|
161
170
|
end
|
162
|
-
|
163
|
-
return unless event.include?(@field) # Skip mapping in case event does not have @event field.
|
164
|
-
return if event.include?(@destination) and not @override # Skip mapping in case @destination field already exists and @override is disabled.
|
165
|
-
|
166
171
|
begin
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
if not matched and @fallback
|
175
|
-
event[@destination] = event.sprintf(@fallback)
|
176
|
-
matched = true
|
172
|
+
matched = true
|
173
|
+
if @override
|
174
|
+
result = event;
|
175
|
+
else
|
176
|
+
event[@result_key] = {}
|
177
|
+
result = event[@result_key]
|
177
178
|
end
|
178
|
-
|
179
|
+
@fields.each { |key|
|
180
|
+
key_string = key.to_s
|
181
|
+
if event.include?(key_string)
|
182
|
+
val_string = event[key_string]
|
183
|
+
if get_map.include?(val_string)
|
184
|
+
result[key_string] = get_map[val_string]
|
185
|
+
else
|
186
|
+
if @default_values and @default_values.include?(key_string)
|
187
|
+
result[key_string] = event.sprintf(@default_values[key_string])
|
188
|
+
end
|
189
|
+
matched = false
|
190
|
+
end
|
191
|
+
end
|
192
|
+
}
|
193
|
+
filter_matched(event) if matched and result.length == @fields.length
|
179
194
|
rescue Exception => e
|
180
|
-
@logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @
|
195
|
+
@logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @fields, :event => event)
|
181
196
|
end
|
182
197
|
end # def filter
|
183
198
|
end # class LogStash::Filters::LookUp
|
@@ -1,13 +1,13 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-filter-lookup'
|
3
|
-
s.version = '
|
3
|
+
s.version = '2.0.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
|
-
s.summary =
|
6
|
-
s.description =
|
7
|
-
s.authors = [
|
5
|
+
s.summary = 'A general search and replace tool which uses a configured web service to determine replacement values.'
|
6
|
+
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-filter-lookup. This gem is not a stand-alone program. See https://github.com/angel9484/logstash-filter-lookup'
|
7
|
+
s.authors = ['Elastic']
|
8
8
|
s.email = 'info@elastic.co'
|
9
|
-
s.homepage =
|
10
|
-
s.require_paths = [
|
9
|
+
s.homepage = 'http://www.elastic.co/guide/en/logstash/current/index.html'
|
10
|
+
s.require_paths = ['lib']
|
11
11
|
|
12
12
|
# Files
|
13
13
|
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
@@ -16,9 +16,10 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
17
17
|
|
18
18
|
# Special flag to let us know this is actually a logstash plugin
|
19
|
-
s.metadata = {
|
19
|
+
s.metadata = {'logstash_plugin' => 'true', 'logstash_group' => 'filter'}
|
20
20
|
|
21
21
|
# Gem dependencies
|
22
|
-
s.add_runtime_dependency
|
22
|
+
s.add_runtime_dependency 'logstash-core', '>= 1.5.0', '< 3.0.0'
|
23
|
+
s.add_runtime_dependency 'rest-client', '>= 1.8.0'
|
23
24
|
s.add_development_dependency 'logstash-devutils', '~> 0'
|
24
25
|
end
|
data/spec/filters/lookup_spec.rb
CHANGED
@@ -2,7 +2,6 @@
|
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/filters/lookup"
|
4
4
|
require "webmock/rspec"
|
5
|
-
require 'digest/sha1'
|
6
5
|
WebMock.disable_net_connect!(allow_localhost: true)
|
7
6
|
|
8
7
|
describe LogStash::Filters::LookUp do
|
@@ -10,126 +9,305 @@ describe LogStash::Filters::LookUp do
|
|
10
9
|
let(:config) { Hash.new }
|
11
10
|
subject { described_class.new(config) }
|
12
11
|
|
12
|
+
describe "file mapping" do
|
13
|
+
config <<-CONFIG
|
14
|
+
filter {
|
15
|
+
lookup {
|
16
|
+
fields => ["status"]
|
17
|
+
type => "file"
|
18
|
+
path => "filename"
|
19
|
+
}
|
20
|
+
}
|
21
|
+
CONFIG
|
22
|
+
content = "\
|
23
|
+
'200': OK\n\
|
24
|
+
'300': Redirect\n\
|
25
|
+
'400': Client Error\n\
|
26
|
+
'500': Server Error"
|
27
|
+
filename = 'filename'
|
28
|
+
|
29
|
+
RSpec.configure do |config|
|
30
|
+
config.before(:each) do
|
31
|
+
allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
|
32
|
+
end
|
33
|
+
end
|
34
|
+
sample({"status" => "200"}) do
|
35
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
13
39
|
describe "webserver mapping" do
|
14
|
-
|
40
|
+
config <<-CONFIG
|
15
41
|
filter {
|
16
42
|
lookup {
|
17
|
-
|
18
|
-
|
19
|
-
|
43
|
+
fields => ["status"]
|
44
|
+
format => "yml"
|
45
|
+
url => "http://dummyurl/"
|
20
46
|
}
|
21
47
|
}
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
stub_request(:get, "http://dummyurl/").
|
29
|
-
with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
|
30
|
-
to_return(:status => 200, :body => "\
|
48
|
+
CONFIG
|
49
|
+
|
50
|
+
RSpec.configure do |config|
|
51
|
+
config.before(:each) do
|
52
|
+
stub_request(:any, "http://dummyurl/").
|
53
|
+
to_return(:status => 200, :body => "\
|
31
54
|
'200': OK\n\
|
32
55
|
'300': Redirect\n\
|
33
56
|
'400': Client Error\n\
|
34
57
|
'500': Server Error", :headers => {})
|
35
|
-
end
|
36
|
-
config.after(:all) do
|
37
|
-
FileUtils.rm_rf(hash+'.yml')
|
38
|
-
end
|
39
58
|
end
|
59
|
+
end
|
40
60
|
|
41
|
-
|
42
|
-
|
43
|
-
|
61
|
+
sample({"status" => "200"}) do
|
62
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
63
|
+
end
|
44
64
|
end
|
45
65
|
|
46
66
|
describe "webserver mapping existing YML" do
|
47
|
-
|
67
|
+
config <<-CONFIG
|
48
68
|
filter {
|
49
69
|
lookup {
|
50
|
-
|
51
|
-
|
52
|
-
|
70
|
+
fields => ["status"]
|
71
|
+
format => "yml"
|
72
|
+
url => "http://dummyurl/"
|
53
73
|
}
|
54
74
|
}
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
File.open(hash+'.yml', 'wb') { |f| f.write("\
|
62
|
-
'200': OKF\n\
|
63
|
-
'300': Redirect\n\
|
64
|
-
'400': Client Error\n\
|
65
|
-
'500': Server Error") }
|
66
|
-
stub_request(:get, "http://dummyurl/").
|
67
|
-
with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
|
68
|
-
to_return(:status => 200, :body => "\
|
75
|
+
CONFIG
|
76
|
+
|
77
|
+
RSpec.configure do |config|
|
78
|
+
config.before(:each) do
|
79
|
+
stub_request(:get, "http://dummyurl/").
|
80
|
+
to_return(:status => 200, :body => "\
|
69
81
|
'200': OK\n\
|
70
82
|
'300': Redirect\n\
|
71
83
|
'400': Client Error\n\
|
72
84
|
'500': Server Error", :headers => {})
|
73
|
-
end
|
74
|
-
config.after(:all) do
|
75
|
-
FileUtils.rm_rf(hash+'.yml')
|
76
|
-
end
|
77
85
|
end
|
86
|
+
end
|
78
87
|
|
79
|
-
|
80
|
-
|
81
|
-
|
88
|
+
sample({"status" => "200"}) do
|
89
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
90
|
+
end
|
82
91
|
end
|
83
92
|
|
84
|
-
describe "webserver mapping not valid" do
|
85
|
-
|
93
|
+
describe "webserver mapping not valid on register" do
|
94
|
+
config <<-CONFIG
|
86
95
|
filter {
|
87
96
|
lookup {
|
88
|
-
|
89
|
-
|
90
|
-
|
97
|
+
fields => ["status"]
|
98
|
+
format => "yml"
|
99
|
+
url => "http://dummyurl/"
|
91
100
|
}
|
92
101
|
}
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
stub_request(:get, "http://dummyurl/").
|
100
|
-
with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
|
101
|
-
to_return(:status => 200, :body => "\
|
102
|
+
CONFIG
|
103
|
+
|
104
|
+
RSpec.configure do |config|
|
105
|
+
config.before(:each) do
|
106
|
+
stub_request(:get, "http://dummyurl/").
|
107
|
+
to_return(:status => 200, :body => "\
|
102
108
|
'200': OK\n\
|
103
109
|
'300': Redirect\n\
|
104
110
|
'400', Client Error\n\
|
105
111
|
'500': Server Error", :headers => {})
|
106
|
-
end
|
107
|
-
config.after(:all) do
|
108
|
-
FileUtils.rm_rf(hash+'.yml')
|
109
|
-
end
|
110
112
|
end
|
113
|
+
end
|
114
|
+
|
115
|
+
sample({"status" => "200"}) do
|
116
|
+
expect { subject }.to raise_error
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
describe "webserver mapping JSON" do
|
121
|
+
config <<-CONFIG
|
122
|
+
filter {
|
123
|
+
lookup {
|
124
|
+
fields => ["status"]
|
125
|
+
format => "json"
|
126
|
+
url => "http://dummyurl/json"
|
127
|
+
}
|
128
|
+
}
|
129
|
+
CONFIG
|
130
|
+
|
131
|
+
RSpec.configure do |config|
|
132
|
+
config.before(:each) do
|
133
|
+
stub_request(:get, "http://dummyurl/json").
|
134
|
+
to_return(:status => 200, :body => '{
|
135
|
+
"200": "OK",
|
136
|
+
"300": "Redirect",
|
137
|
+
"400": "Client Error",
|
138
|
+
"500": "Server Error"
|
139
|
+
}', :headers => {})
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
sample({"status" => "200"}) do
|
144
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
describe "webserver mapping existing JSON" do
|
149
|
+
config <<-CONFIG
|
150
|
+
filter {
|
151
|
+
lookup {
|
152
|
+
fields => ["status"]
|
153
|
+
format => "json"
|
154
|
+
url => "http://dummyurl/json"
|
155
|
+
}
|
156
|
+
}
|
157
|
+
CONFIG
|
158
|
+
|
159
|
+
RSpec.configure do |config|
|
160
|
+
config.before(:each) do
|
161
|
+
stub_request(:get, "http://dummyurl/json").
|
162
|
+
to_return(:status => 200, :body => '{
|
163
|
+
"200": "OK",
|
164
|
+
"300": "Redirect",
|
165
|
+
"400": "Client Error",
|
166
|
+
"500": "Server Error"
|
167
|
+
}', :headers => {})
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
sample({"status" => "200"}) do
|
172
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
describe "webserver mapping not valid JSON on register" do
|
177
|
+
config <<-CONFIG
|
178
|
+
filter {
|
179
|
+
lookup {
|
180
|
+
fields => ["status"]
|
181
|
+
format => "json"
|
182
|
+
url => "http://dummyurl/json"
|
183
|
+
}
|
184
|
+
}
|
185
|
+
CONFIG
|
186
|
+
|
187
|
+
RSpec.configure do |config|
|
188
|
+
config.before(:each) do
|
189
|
+
stub_request(:get, "http://dummyurl/json").
|
190
|
+
to_return(:status => 200, :body => '{
|
191
|
+
"200": "OK",
|
192
|
+
"300": "Redirect",
|
193
|
+
"400": "Client Error",
|
194
|
+
"500", "Server Error"
|
195
|
+
}', :headers => {})
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
sample({"status" => "200"}) do
|
200
|
+
expect { subject }.to raise_error
|
201
|
+
end
|
202
|
+
end
|
111
203
|
|
112
|
-
|
113
|
-
|
204
|
+
describe "webserver mapping CSV" do
|
205
|
+
config <<-CONFIG
|
206
|
+
filter {
|
207
|
+
lookup {
|
208
|
+
fields => ["status"]
|
209
|
+
format => "csv"
|
210
|
+
url => "http://dummyurl/csv"
|
211
|
+
}
|
212
|
+
}
|
213
|
+
CONFIG
|
214
|
+
|
215
|
+
RSpec.configure do |config|
|
216
|
+
config.before(:each) do
|
217
|
+
stub_request(:get, "http://dummyurl/csv").
|
218
|
+
to_return(:status => 200, :body => "200,OK
|
219
|
+
300,Redirect
|
220
|
+
400,Client Error
|
221
|
+
500,Server Error
|
222
|
+
", :headers => {})
|
114
223
|
end
|
224
|
+
end
|
225
|
+
|
226
|
+
sample({"status" => "200"}) do
|
227
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
228
|
+
end
|
115
229
|
end
|
116
230
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
231
|
+
describe "webserver mapping existing CSV" do
|
232
|
+
config <<-CONFIG
|
233
|
+
filter {
|
234
|
+
lookup {
|
235
|
+
fields => ["status"]
|
236
|
+
format => "csv"
|
237
|
+
url => "http://dummyurl/csv"
|
238
|
+
}
|
239
|
+
}
|
240
|
+
CONFIG
|
241
|
+
|
242
|
+
RSpec.configure do |config|
|
243
|
+
config.before(:each) do
|
244
|
+
stub_request(:get, "http://dummyurl/csv").
|
245
|
+
to_return(:status => 200, :body => "200,OK
|
246
|
+
300,Redirect
|
247
|
+
400,Client Error
|
248
|
+
500,Server Error
|
249
|
+
", :headers => {})
|
125
250
|
end
|
251
|
+
end
|
252
|
+
|
253
|
+
sample({"status" => "200"}) do
|
254
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
255
|
+
end
|
256
|
+
end
|
126
257
|
|
127
|
-
|
258
|
+
describe "webserver mapping not valid CSV" do
|
259
|
+
config <<-CONFIG
|
260
|
+
filter {
|
261
|
+
lookup {
|
262
|
+
fields => ["status"]
|
263
|
+
format => "csv"
|
264
|
+
url => "http://dummyurl/csv"
|
265
|
+
}
|
266
|
+
}
|
267
|
+
CONFIG
|
128
268
|
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
269
|
+
RSpec.configure do |config|
|
270
|
+
config.before(:each) do
|
271
|
+
stub_request(:get, "http://dummyurl/csv").
|
272
|
+
to_return(:status => 200, :body => "200OK
|
273
|
+
300,Redirect
|
274
|
+
400,Client Error
|
275
|
+
500:Server Error
|
276
|
+
", :headers => {})
|
133
277
|
end
|
134
278
|
end
|
279
|
+
|
280
|
+
sample({"status" => "200"}) do
|
281
|
+
insist { subject["lookup_result"]["status"] } == nil
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
context "allow sprintf" do
|
286
|
+
let(:config) do
|
287
|
+
{
|
288
|
+
"fields" => ["status"],
|
289
|
+
"default_values" => {"status"=>"%{missing_mapping}"},
|
290
|
+
"type" => "file",
|
291
|
+
"path"=>"filename"
|
292
|
+
}
|
293
|
+
end
|
294
|
+
content = "\
|
295
|
+
'200': OK\n\
|
296
|
+
'300': Redirect\n\
|
297
|
+
'400': Client Error\n\
|
298
|
+
'500': Server Error"
|
299
|
+
filename = 'filename'
|
300
|
+
RSpec.configure do |config|
|
301
|
+
config.before(:each) do
|
302
|
+
allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
|
303
|
+
end
|
304
|
+
end
|
305
|
+
let(:event) { LogStash::Event.new("status" => "250", "missing_mapping" => "missing no match") }
|
306
|
+
|
307
|
+
it "return the exact mapping" do
|
308
|
+
subject.register
|
309
|
+
subject.filter(event)
|
310
|
+
expect(event["lookup_result"]["status"]).to eq("missing no match")
|
311
|
+
end
|
312
|
+
end
|
135
313
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-lookup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-03-
|
11
|
+
date: 2016-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -30,6 +30,20 @@ dependencies:
|
|
30
30
|
- - <
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 3.0.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: 1.8.0
|
39
|
+
name: rest-client
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - '>='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 1.8.0
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
requirement: !ruby/object:Gem::Requirement
|
35
49
|
requirements:
|