logstash-filter-lookup 1.2.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/README.md +2 -3
- data/lib/logstash/filters/lookup.rb +97 -82
- data/logstash-filter-lookup.gemspec +9 -8
- data/spec/filters/lookup_spec.rb +257 -79
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 175c91bd50e708f7d89b7db370bac124f3770d60
|
4
|
+
data.tar.gz: fb814d76271fa06046fab307df9ad0fc169d684d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9385b8dd385a3fe54b2908b81e7cab60179df6d5b9e26e1ecdab131b690baf56d94847ed7e8bc8ceac45889b6d1c6382b0e7d67ab310d41506ea54c7650a805a
|
7
|
+
data.tar.gz: 063b615c61acc2d109b97496b6f094b90ff4eb037a24517939902ad0d6b75297ca0ed338965d64b83d918d80f111d8fd15e9224c60f0402c6b570e2d24f474b1
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -1,8 +1,7 @@
|
|
1
|
-
#
|
1
|
+
# logstash-filter-lookup
|
2
2
|
|
3
|
-
|
3
|
+
New lookup plugin for dynamic lookup enrichment from multiple types of sources. There are a few different filters out there today that conduct different types of lookups, which may make sense being encapsulated within one lookup filter.
|
4
4
|
|
5
|
-
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
6
5
|
|
7
6
|
## Example
|
8
7
|
### 1. Web Service Example
|
@@ -1,10 +1,10 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'logstash/filters/base'
|
3
3
|
require 'logstash/namespace'
|
4
|
-
require 'open-uri'
|
5
|
-
require 'digest/sha1'
|
6
4
|
require 'json'
|
7
5
|
require 'csv'
|
6
|
+
require 'rest_client'
|
7
|
+
|
8
8
|
|
9
9
|
# A general search and replace tool which uses a Web service with a YAML, CSV or JSON response to determine replacement values.
|
10
10
|
#
|
@@ -14,10 +14,7 @@ require 'csv'
|
|
14
14
|
# matches the EXACT contents of a map entry key, the field's value will be substituted
|
15
15
|
# with the matched key's value from the map.
|
16
16
|
#
|
17
|
-
#
|
18
|
-
# maching event field (in-place). However, by using the `destination`
|
19
|
-
# configuration item, you may also specify a target event field to
|
20
|
-
# populate with the new mapd value.
|
17
|
+
# The lookup filter will put the contents of the maching event fields into field result.
|
21
18
|
|
22
19
|
class LogStash::Filters::LookUp < LogStash::Filters::Base
|
23
20
|
config_name "lookup"
|
@@ -25,56 +22,59 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base
|
|
25
22
|
# match by the map filter (e.g. `message`, `host`, `response_code`).
|
26
23
|
#
|
27
24
|
# If this field is an array, only the first value will be used.
|
28
|
-
config :
|
25
|
+
config :fields, :validate => :array, :required => true
|
29
26
|
|
30
|
-
#
|
31
|
-
#
|
32
|
-
# value with the new mapping value.
|
27
|
+
# By default false, if false, then all the result will be stored into a field called lookup_result
|
28
|
+
# If true, the same field used to lookup, will be replaced.
|
33
29
|
config :override, :validate => :boolean, :default => false
|
34
30
|
|
31
|
+
# Can be webservice or file
|
32
|
+
config :type, :validate => :string, :default => 'webservice'
|
33
|
+
|
34
|
+
# Path of a file
|
35
|
+
config :path, :validate => :string
|
36
|
+
|
37
|
+
config :format, :validate => :string, :default => 'yml'
|
38
|
+
|
39
|
+
config :headers, :validate => :hash, :default => { }
|
40
|
+
config :params, :validate => :hash, :default => { }
|
41
|
+
|
42
|
+
#default field where store results if override option is false
|
43
|
+
config :result_key, :validate => :string, :default => "lookup_result"
|
44
|
+
|
35
45
|
# The full URI path of a Web service who generates an JSON, yml or CSV format response.
|
36
46
|
# requires to append as suffix the format type. Ex: http://localhost:8080/geoPoints?type=json
|
37
47
|
# http://localhost:8080/geoPoints/json
|
38
48
|
# http://localhost:8080/geoPoints/csv
|
39
49
|
# If no suffix matches, defaults to YAML
|
40
|
-
config :
|
50
|
+
config :url, :validate => :string
|
51
|
+
|
52
|
+
#HTTP method
|
53
|
+
config :method, :validate => :string, :default => "get"
|
54
|
+
|
55
|
+
|
41
56
|
|
42
57
|
# When using a map file or url, this setting will indicate how frequently
|
43
58
|
# (in seconds) logstash will check the YAML file or url for updates.
|
44
59
|
config :refresh_interval, :validate => :number, :default => 300
|
45
60
|
|
46
|
-
# The destination field you wish to populate with the mapd code. The default
|
47
|
-
# is a field named `mapping`. Set this to the same value as source if you want
|
48
|
-
# to do a substitution, in this case filter will allways succeed. This will clobber
|
49
|
-
# the old value of the source field!
|
50
|
-
config :destination, :validate => :string, :default => "mapping"
|
51
|
-
|
52
61
|
# In case no mapping occurs in the event (no matches), this will add a default
|
53
62
|
# mapping string, which will always populate `field`, if the match failed.
|
54
|
-
|
55
|
-
# For example, if we have configured `fallback => "no match"`, using this map:
|
56
|
-
# [source,ruby]
|
57
|
-
# foo: bar
|
58
|
-
#
|
59
|
-
# Then, if logstash received an event with the field `foo` set to `bar`, the destination
|
60
|
-
# field would be set to `bar`. However, if logstash received an event with `foo` set to `nope`,
|
61
|
-
# then the destination field would still be populated, but with the value of `no match`.
|
62
|
-
# This configuration can be dynamic and include parts of the event using the `%{field}` syntax.
|
63
|
-
config :fallback, :validate => :string
|
63
|
+
config :default_values, :validate => :array, :default => {}
|
64
64
|
|
65
65
|
def get_map
|
66
66
|
@my_map
|
67
67
|
end
|
68
68
|
|
69
|
-
def
|
70
|
-
|
69
|
+
def fill_map(data)
|
70
|
+
get_map.merge!(data)
|
71
71
|
end
|
72
72
|
|
73
73
|
public
|
74
74
|
def register
|
75
75
|
@my_map = {}
|
76
76
|
@next_refresh = Time.now + @refresh_interval
|
77
|
-
|
77
|
+
load(true)
|
78
78
|
@logger.debug? and @logger.debug("#{self.class.name}: map - ", :map => get_map)
|
79
79
|
type = 'Exact'
|
80
80
|
@logger.debug? and @logger.debug("#{self.class.name}: map mapping method - "+type)
|
@@ -83,101 +83,116 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base
|
|
83
83
|
# def register
|
84
84
|
|
85
85
|
def json_loader(data)
|
86
|
-
|
86
|
+
fill_map(JSON.parse(data))
|
87
87
|
end
|
88
88
|
|
89
89
|
def csv_loader(data)
|
90
|
-
data = CSV.
|
90
|
+
data = CSV.parse(data).inject(Hash.new) do |acc, v|
|
91
91
|
acc[v[0]] = v[1]
|
92
92
|
acc
|
93
93
|
end
|
94
|
-
|
94
|
+
fill_map(data)
|
95
95
|
end
|
96
96
|
|
97
97
|
def yml_loader(data)
|
98
|
-
|
98
|
+
fill_map(YAML.load(data))
|
99
99
|
end
|
100
100
|
|
101
|
-
def
|
101
|
+
def load_data(registering, data)
|
102
102
|
begin
|
103
|
-
if
|
103
|
+
if @format.eql?('json')
|
104
104
|
return json_loader(data)
|
105
|
-
elsif
|
105
|
+
elsif @format.eql?('csv')
|
106
106
|
return csv_loader(data)
|
107
107
|
end
|
108
108
|
yml_loader(data)
|
109
109
|
rescue Exception => _
|
110
110
|
if registering
|
111
|
-
raise "#{self.class.name}: Bad Syntax in
|
111
|
+
raise "#{self.class.name}: Bad Syntax in data #{data}"
|
112
112
|
else
|
113
|
-
@logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path =>
|
113
|
+
@logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => data)
|
114
114
|
end
|
115
115
|
end
|
116
116
|
end
|
117
117
|
|
118
|
-
def
|
119
|
-
if path.end_with?('json')
|
120
|
-
return '.json'
|
121
|
-
elsif path.end_with?('csv')
|
122
|
-
return '.csv'
|
123
|
-
end
|
124
|
-
'.yml'
|
125
|
-
end
|
126
|
-
|
127
|
-
public
|
128
|
-
def download_ws(path, registering=false)
|
129
|
-
extension = get_extension(path)
|
130
|
-
temp_extension = '_temp'+extension;
|
131
|
-
file_name = Digest::SHA1.hexdigest path
|
118
|
+
def load(registering=false)
|
132
119
|
begin
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
120
|
+
if @type=='webservice'
|
121
|
+
route = @url
|
122
|
+
data = get_webservice_content(route)
|
123
|
+
elsif @type=='file'
|
124
|
+
route = @path
|
125
|
+
data = get_file_content(route)
|
137
126
|
end
|
127
|
+
load_data(registering, data)
|
138
128
|
rescue Exception => _
|
139
129
|
if registering
|
140
|
-
raise "#{self.class.name}: Failed to initialize with #{
|
130
|
+
raise "#{self.class.name}: Failed to initialize with type #{type} and route #{route}"
|
141
131
|
end
|
142
|
-
@logger.warn("#{self.class.name}: Something happened with
|
132
|
+
@logger.warn("#{self.class.name}: Something happened with load. Continuing with old map", :type => @type, :route => route)
|
143
133
|
end
|
134
|
+
end
|
144
135
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
136
|
+
# def load
|
137
|
+
|
138
|
+
def get_file_content(file)
|
139
|
+
data = ''
|
140
|
+
File.open(file, 'r') do |read_file|
|
141
|
+
data +=read_file.read
|
142
|
+
end
|
143
|
+
data
|
144
|
+
end
|
145
|
+
|
146
|
+
# def get_file_content
|
147
|
+
|
148
|
+
def get_webservice_content(path)
|
149
|
+
case @method
|
150
|
+
when "get"
|
151
|
+
data = RestClient.get sprint(path), sprint(@headers)
|
152
|
+
when "post"
|
153
|
+
data = RestClient.post sprint(path), sprint(@params), sprint(@headers)
|
150
154
|
end
|
155
|
+
data
|
151
156
|
end
|
152
157
|
|
153
|
-
# def
|
158
|
+
# def get_webservice_content
|
159
|
+
|
160
|
+
def sprint(hash)
|
161
|
+
hash
|
162
|
+
end
|
154
163
|
|
155
164
|
public
|
156
165
|
def filter(event)
|
157
166
|
if @next_refresh < Time.now
|
158
|
-
|
167
|
+
load()
|
159
168
|
@next_refresh = Time.now + @refresh_interval
|
160
169
|
@logger.info('downloading and refreshing map file')
|
161
170
|
end
|
162
|
-
|
163
|
-
return unless event.include?(@field) # Skip mapping in case event does not have @event field.
|
164
|
-
return if event.include?(@destination) and not @override # Skip mapping in case @destination field already exists and @override is disabled.
|
165
|
-
|
166
171
|
begin
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
if not matched and @fallback
|
175
|
-
event[@destination] = event.sprintf(@fallback)
|
176
|
-
matched = true
|
172
|
+
matched = true
|
173
|
+
if @override
|
174
|
+
result = event;
|
175
|
+
else
|
176
|
+
event[@result_key] = {}
|
177
|
+
result = event[@result_key]
|
177
178
|
end
|
178
|
-
|
179
|
+
@fields.each { |key|
|
180
|
+
key_string = key.to_s
|
181
|
+
if event.include?(key_string)
|
182
|
+
val_string = event[key_string]
|
183
|
+
if get_map.include?(val_string)
|
184
|
+
result[key_string] = get_map[val_string]
|
185
|
+
else
|
186
|
+
if @default_values and @default_values.include?(key_string)
|
187
|
+
result[key_string] = event.sprintf(@default_values[key_string])
|
188
|
+
end
|
189
|
+
matched = false
|
190
|
+
end
|
191
|
+
end
|
192
|
+
}
|
193
|
+
filter_matched(event) if matched and result.length == @fields.length
|
179
194
|
rescue Exception => e
|
180
|
-
@logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @
|
195
|
+
@logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @fields, :event => event)
|
181
196
|
end
|
182
197
|
end # def filter
|
183
198
|
end # class LogStash::Filters::LookUp
|
@@ -1,13 +1,13 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-filter-lookup'
|
3
|
-
s.version = '
|
3
|
+
s.version = '2.0.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
|
-
s.summary =
|
6
|
-
s.description =
|
7
|
-
s.authors = [
|
5
|
+
s.summary = 'A general search and replace tool which uses a configured web service to determine replacement values.'
|
6
|
+
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-filter-lookup. This gem is not a stand-alone program. See https://github.com/angel9484/logstash-filter-lookup'
|
7
|
+
s.authors = ['Elastic']
|
8
8
|
s.email = 'info@elastic.co'
|
9
|
-
s.homepage =
|
10
|
-
s.require_paths = [
|
9
|
+
s.homepage = 'http://www.elastic.co/guide/en/logstash/current/index.html'
|
10
|
+
s.require_paths = ['lib']
|
11
11
|
|
12
12
|
# Files
|
13
13
|
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
@@ -16,9 +16,10 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
17
17
|
|
18
18
|
# Special flag to let us know this is actually a logstash plugin
|
19
|
-
s.metadata = {
|
19
|
+
s.metadata = {'logstash_plugin' => 'true', 'logstash_group' => 'filter'}
|
20
20
|
|
21
21
|
# Gem dependencies
|
22
|
-
s.add_runtime_dependency
|
22
|
+
s.add_runtime_dependency 'logstash-core', '>= 1.5.0', '< 3.0.0'
|
23
|
+
s.add_runtime_dependency 'rest-client', '>= 1.8.0'
|
23
24
|
s.add_development_dependency 'logstash-devutils', '~> 0'
|
24
25
|
end
|
data/spec/filters/lookup_spec.rb
CHANGED
@@ -2,7 +2,6 @@
|
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/filters/lookup"
|
4
4
|
require "webmock/rspec"
|
5
|
-
require 'digest/sha1'
|
6
5
|
WebMock.disable_net_connect!(allow_localhost: true)
|
7
6
|
|
8
7
|
describe LogStash::Filters::LookUp do
|
@@ -10,126 +9,305 @@ describe LogStash::Filters::LookUp do
|
|
10
9
|
let(:config) { Hash.new }
|
11
10
|
subject { described_class.new(config) }
|
12
11
|
|
12
|
+
describe "file mapping" do
|
13
|
+
config <<-CONFIG
|
14
|
+
filter {
|
15
|
+
lookup {
|
16
|
+
fields => ["status"]
|
17
|
+
type => "file"
|
18
|
+
path => "filename"
|
19
|
+
}
|
20
|
+
}
|
21
|
+
CONFIG
|
22
|
+
content = "\
|
23
|
+
'200': OK\n\
|
24
|
+
'300': Redirect\n\
|
25
|
+
'400': Client Error\n\
|
26
|
+
'500': Server Error"
|
27
|
+
filename = 'filename'
|
28
|
+
|
29
|
+
RSpec.configure do |config|
|
30
|
+
config.before(:each) do
|
31
|
+
allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
|
32
|
+
end
|
33
|
+
end
|
34
|
+
sample({"status" => "200"}) do
|
35
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
13
39
|
describe "webserver mapping" do
|
14
|
-
|
40
|
+
config <<-CONFIG
|
15
41
|
filter {
|
16
42
|
lookup {
|
17
|
-
|
18
|
-
|
19
|
-
|
43
|
+
fields => ["status"]
|
44
|
+
format => "yml"
|
45
|
+
url => "http://dummyurl/"
|
20
46
|
}
|
21
47
|
}
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
stub_request(:get, "http://dummyurl/").
|
29
|
-
with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
|
30
|
-
to_return(:status => 200, :body => "\
|
48
|
+
CONFIG
|
49
|
+
|
50
|
+
RSpec.configure do |config|
|
51
|
+
config.before(:each) do
|
52
|
+
stub_request(:any, "http://dummyurl/").
|
53
|
+
to_return(:status => 200, :body => "\
|
31
54
|
'200': OK\n\
|
32
55
|
'300': Redirect\n\
|
33
56
|
'400': Client Error\n\
|
34
57
|
'500': Server Error", :headers => {})
|
35
|
-
end
|
36
|
-
config.after(:all) do
|
37
|
-
FileUtils.rm_rf(hash+'.yml')
|
38
|
-
end
|
39
58
|
end
|
59
|
+
end
|
40
60
|
|
41
|
-
|
42
|
-
|
43
|
-
|
61
|
+
sample({"status" => "200"}) do
|
62
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
63
|
+
end
|
44
64
|
end
|
45
65
|
|
46
66
|
describe "webserver mapping existing YML" do
|
47
|
-
|
67
|
+
config <<-CONFIG
|
48
68
|
filter {
|
49
69
|
lookup {
|
50
|
-
|
51
|
-
|
52
|
-
|
70
|
+
fields => ["status"]
|
71
|
+
format => "yml"
|
72
|
+
url => "http://dummyurl/"
|
53
73
|
}
|
54
74
|
}
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
File.open(hash+'.yml', 'wb') { |f| f.write("\
|
62
|
-
'200': OKF\n\
|
63
|
-
'300': Redirect\n\
|
64
|
-
'400': Client Error\n\
|
65
|
-
'500': Server Error") }
|
66
|
-
stub_request(:get, "http://dummyurl/").
|
67
|
-
with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
|
68
|
-
to_return(:status => 200, :body => "\
|
75
|
+
CONFIG
|
76
|
+
|
77
|
+
RSpec.configure do |config|
|
78
|
+
config.before(:each) do
|
79
|
+
stub_request(:get, "http://dummyurl/").
|
80
|
+
to_return(:status => 200, :body => "\
|
69
81
|
'200': OK\n\
|
70
82
|
'300': Redirect\n\
|
71
83
|
'400': Client Error\n\
|
72
84
|
'500': Server Error", :headers => {})
|
73
|
-
end
|
74
|
-
config.after(:all) do
|
75
|
-
FileUtils.rm_rf(hash+'.yml')
|
76
|
-
end
|
77
85
|
end
|
86
|
+
end
|
78
87
|
|
79
|
-
|
80
|
-
|
81
|
-
|
88
|
+
sample({"status" => "200"}) do
|
89
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
90
|
+
end
|
82
91
|
end
|
83
92
|
|
84
|
-
describe "webserver mapping not valid" do
|
85
|
-
|
93
|
+
describe "webserver mapping not valid on register" do
|
94
|
+
config <<-CONFIG
|
86
95
|
filter {
|
87
96
|
lookup {
|
88
|
-
|
89
|
-
|
90
|
-
|
97
|
+
fields => ["status"]
|
98
|
+
format => "yml"
|
99
|
+
url => "http://dummyurl/"
|
91
100
|
}
|
92
101
|
}
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
stub_request(:get, "http://dummyurl/").
|
100
|
-
with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
|
101
|
-
to_return(:status => 200, :body => "\
|
102
|
+
CONFIG
|
103
|
+
|
104
|
+
RSpec.configure do |config|
|
105
|
+
config.before(:each) do
|
106
|
+
stub_request(:get, "http://dummyurl/").
|
107
|
+
to_return(:status => 200, :body => "\
|
102
108
|
'200': OK\n\
|
103
109
|
'300': Redirect\n\
|
104
110
|
'400', Client Error\n\
|
105
111
|
'500': Server Error", :headers => {})
|
106
|
-
end
|
107
|
-
config.after(:all) do
|
108
|
-
FileUtils.rm_rf(hash+'.yml')
|
109
|
-
end
|
110
112
|
end
|
113
|
+
end
|
114
|
+
|
115
|
+
sample({"status" => "200"}) do
|
116
|
+
expect { subject }.to raise_error
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
describe "webserver mapping JSON" do
|
121
|
+
config <<-CONFIG
|
122
|
+
filter {
|
123
|
+
lookup {
|
124
|
+
fields => ["status"]
|
125
|
+
format => "json"
|
126
|
+
url => "http://dummyurl/json"
|
127
|
+
}
|
128
|
+
}
|
129
|
+
CONFIG
|
130
|
+
|
131
|
+
RSpec.configure do |config|
|
132
|
+
config.before(:each) do
|
133
|
+
stub_request(:get, "http://dummyurl/json").
|
134
|
+
to_return(:status => 200, :body => '{
|
135
|
+
"200": "OK",
|
136
|
+
"300": "Redirect",
|
137
|
+
"400": "Client Error",
|
138
|
+
"500": "Server Error"
|
139
|
+
}', :headers => {})
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
sample({"status" => "200"}) do
|
144
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
describe "webserver mapping existing JSON" do
|
149
|
+
config <<-CONFIG
|
150
|
+
filter {
|
151
|
+
lookup {
|
152
|
+
fields => ["status"]
|
153
|
+
format => "json"
|
154
|
+
url => "http://dummyurl/json"
|
155
|
+
}
|
156
|
+
}
|
157
|
+
CONFIG
|
158
|
+
|
159
|
+
RSpec.configure do |config|
|
160
|
+
config.before(:each) do
|
161
|
+
stub_request(:get, "http://dummyurl/json").
|
162
|
+
to_return(:status => 200, :body => '{
|
163
|
+
"200": "OK",
|
164
|
+
"300": "Redirect",
|
165
|
+
"400": "Client Error",
|
166
|
+
"500": "Server Error"
|
167
|
+
}', :headers => {})
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
sample({"status" => "200"}) do
|
172
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
describe "webserver mapping not valid JSON on register" do
|
177
|
+
config <<-CONFIG
|
178
|
+
filter {
|
179
|
+
lookup {
|
180
|
+
fields => ["status"]
|
181
|
+
format => "json"
|
182
|
+
url => "http://dummyurl/json"
|
183
|
+
}
|
184
|
+
}
|
185
|
+
CONFIG
|
186
|
+
|
187
|
+
RSpec.configure do |config|
|
188
|
+
config.before(:each) do
|
189
|
+
stub_request(:get, "http://dummyurl/json").
|
190
|
+
to_return(:status => 200, :body => '{
|
191
|
+
"200": "OK",
|
192
|
+
"300": "Redirect",
|
193
|
+
"400": "Client Error",
|
194
|
+
"500", "Server Error"
|
195
|
+
}', :headers => {})
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
sample({"status" => "200"}) do
|
200
|
+
expect { subject }.to raise_error
|
201
|
+
end
|
202
|
+
end
|
111
203
|
|
112
|
-
|
113
|
-
|
204
|
+
describe "webserver mapping CSV" do
|
205
|
+
config <<-CONFIG
|
206
|
+
filter {
|
207
|
+
lookup {
|
208
|
+
fields => ["status"]
|
209
|
+
format => "csv"
|
210
|
+
url => "http://dummyurl/csv"
|
211
|
+
}
|
212
|
+
}
|
213
|
+
CONFIG
|
214
|
+
|
215
|
+
RSpec.configure do |config|
|
216
|
+
config.before(:each) do
|
217
|
+
stub_request(:get, "http://dummyurl/csv").
|
218
|
+
to_return(:status => 200, :body => "200,OK
|
219
|
+
300,Redirect
|
220
|
+
400,Client Error
|
221
|
+
500,Server Error
|
222
|
+
", :headers => {})
|
114
223
|
end
|
224
|
+
end
|
225
|
+
|
226
|
+
sample({"status" => "200"}) do
|
227
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
228
|
+
end
|
115
229
|
end
|
116
230
|
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
231
|
+
describe "webserver mapping existing CSV" do
|
232
|
+
config <<-CONFIG
|
233
|
+
filter {
|
234
|
+
lookup {
|
235
|
+
fields => ["status"]
|
236
|
+
format => "csv"
|
237
|
+
url => "http://dummyurl/csv"
|
238
|
+
}
|
239
|
+
}
|
240
|
+
CONFIG
|
241
|
+
|
242
|
+
RSpec.configure do |config|
|
243
|
+
config.before(:each) do
|
244
|
+
stub_request(:get, "http://dummyurl/csv").
|
245
|
+
to_return(:status => 200, :body => "200,OK
|
246
|
+
300,Redirect
|
247
|
+
400,Client Error
|
248
|
+
500,Server Error
|
249
|
+
", :headers => {})
|
125
250
|
end
|
251
|
+
end
|
252
|
+
|
253
|
+
sample({"status" => "200"}) do
|
254
|
+
insist { subject["lookup_result"]["status"] } == "OK"
|
255
|
+
end
|
256
|
+
end
|
126
257
|
|
127
|
-
|
258
|
+
describe "webserver mapping not valid CSV" do
|
259
|
+
config <<-CONFIG
|
260
|
+
filter {
|
261
|
+
lookup {
|
262
|
+
fields => ["status"]
|
263
|
+
format => "csv"
|
264
|
+
url => "http://dummyurl/csv"
|
265
|
+
}
|
266
|
+
}
|
267
|
+
CONFIG
|
128
268
|
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
269
|
+
RSpec.configure do |config|
|
270
|
+
config.before(:each) do
|
271
|
+
stub_request(:get, "http://dummyurl/csv").
|
272
|
+
to_return(:status => 200, :body => "200OK
|
273
|
+
300,Redirect
|
274
|
+
400,Client Error
|
275
|
+
500:Server Error
|
276
|
+
", :headers => {})
|
133
277
|
end
|
134
278
|
end
|
279
|
+
|
280
|
+
sample({"status" => "200"}) do
|
281
|
+
insist { subject["lookup_result"]["status"] } == nil
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
context "allow sprintf" do
|
286
|
+
let(:config) do
|
287
|
+
{
|
288
|
+
"fields" => ["status"],
|
289
|
+
"default_values" => {"status"=>"%{missing_mapping}"},
|
290
|
+
"type" => "file",
|
291
|
+
"path"=>"filename"
|
292
|
+
}
|
293
|
+
end
|
294
|
+
content = "\
|
295
|
+
'200': OK\n\
|
296
|
+
'300': Redirect\n\
|
297
|
+
'400': Client Error\n\
|
298
|
+
'500': Server Error"
|
299
|
+
filename = 'filename'
|
300
|
+
RSpec.configure do |config|
|
301
|
+
config.before(:each) do
|
302
|
+
allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
|
303
|
+
end
|
304
|
+
end
|
305
|
+
let(:event) { LogStash::Event.new("status" => "250", "missing_mapping" => "missing no match") }
|
306
|
+
|
307
|
+
it "return the exact mapping" do
|
308
|
+
subject.register
|
309
|
+
subject.filter(event)
|
310
|
+
expect(event["lookup_result"]["status"]).to eq("missing no match")
|
311
|
+
end
|
312
|
+
end
|
135
313
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-lookup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-03-
|
11
|
+
date: 2016-03-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -30,6 +30,20 @@ dependencies:
|
|
30
30
|
- - <
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 3.0.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: 1.8.0
|
39
|
+
name: rest-client
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - '>='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 1.8.0
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
requirement: !ruby/object:Gem::Requirement
|
35
49
|
requirements:
|