logstash-filter-lookup 1.2.1 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8db20eba2cf5828897c358412d6b0d406844239e
4
- data.tar.gz: 364d5cb9dd19ae95d269b23134fe20bb7f797e9c
3
+ metadata.gz: 175c91bd50e708f7d89b7db370bac124f3770d60
4
+ data.tar.gz: fb814d76271fa06046fab307df9ad0fc169d684d
5
5
  SHA512:
6
- metadata.gz: f15abca4bda7c2dcddd29c5c5d3e0002f5720adeda71487339980ccaf55bad5229180b75021662c98fa755eb9c124e2b18261e5be5b98637a86bce60e6fe5f81
7
- data.tar.gz: 87565c30b7bcecae8add5082bae3f22be5f53c9274168d24ae5ac78ad858627db1bbe6d76e85132b4f2d93953975c5e5c47481499ed72ad75f45d3c8eaa82fdd
6
+ metadata.gz: 9385b8dd385a3fe54b2908b81e7cab60179df6d5b9e26e1ecdab131b690baf56d94847ed7e8bc8ceac45889b6d1c6382b0e7d67ab310d41506ea54c7650a805a
7
+ data.tar.gz: 063b615c61acc2d109b97496b6f094b90ff4eb037a24517939902ad0d6b75297ca0ed338965d64b83d918d80f111d8fd15e9224c60f0402c6b570e2d24f474b1
@@ -1,2 +1,5 @@
1
1
  ## 1.0
2
- Initialized logstash-filter-lookup. Formerly called logstash-filter-webservicemap
2
+ Initialized logstash-filter-lookup. Formerly called logstash-filter-webservicemap
3
+
4
+ ## 2.0
5
+ Now can load a file. Uses RestClient
data/README.md CHANGED
@@ -1,8 +1,7 @@
1
- # Logstash Plugin
1
+ # logstash-filter-lookup
2
2
 
3
- This is a plugin for [Logstash](https://github.com/elastic/logstash).
3
+ New lookup plugin for dynamic lookup enrichment from multiple types of sources. There are a few different filters out there today that conduct different types of lookups, which may make sense being encapsulated within one lookup filter.
4
4
 
5
- It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
6
5
 
7
6
  ## Example
8
7
  ### 1. Web Service Example
@@ -1,10 +1,10 @@
1
1
  # encoding: utf-8
2
2
  require 'logstash/filters/base'
3
3
  require 'logstash/namespace'
4
- require 'open-uri'
5
- require 'digest/sha1'
6
4
  require 'json'
7
5
  require 'csv'
6
+ require 'rest_client'
7
+
8
8
 
9
9
  # A general search and replace tool which uses a Web service with a YAML, CSV or JSON response to determine replacement values.
10
10
  #
@@ -14,10 +14,7 @@ require 'csv'
14
14
  # matches the EXACT contents of a map entry key, the field's value will be substituted
15
15
  # with the matched key's value from the map.
16
16
  #
17
- # By default, the lookup filter will replace the contents of the
18
- # maching event field (in-place). However, by using the `destination`
19
- # configuration item, you may also specify a target event field to
20
- # populate with the new mapd value.
17
+ # The lookup filter will put the contents of the maching event fields into field result.
21
18
 
22
19
  class LogStash::Filters::LookUp < LogStash::Filters::Base
23
20
  config_name "lookup"
@@ -25,56 +22,59 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base
25
22
  # match by the map filter (e.g. `message`, `host`, `response_code`).
26
23
  #
27
24
  # If this field is an array, only the first value will be used.
28
- config :field, :validate => :string, :required => true
25
+ config :fields, :validate => :array, :required => true
29
26
 
30
- # If the destination (or target) field already exists, this configuration item specifies
31
- # whether the filter should skip mapping (default) or overwrite the target field
32
- # value with the new mapping value.
27
+ # By default false, if false, then all the result will be stored into a field called lookup_result
28
+ # If true, the same field used to lookup, will be replaced.
33
29
  config :override, :validate => :boolean, :default => false
34
30
 
31
+ # Can be webservice or file
32
+ config :type, :validate => :string, :default => 'webservice'
33
+
34
+ # Path of a file
35
+ config :path, :validate => :string
36
+
37
+ config :format, :validate => :string, :default => 'yml'
38
+
39
+ config :headers, :validate => :hash, :default => { }
40
+ config :params, :validate => :hash, :default => { }
41
+
42
+ #default field where store results if override option is false
43
+ config :result_key, :validate => :string, :default => "lookup_result"
44
+
35
45
  # The full URI path of a Web service who generates an JSON, yml or CSV format response.
36
46
  # requires to append as suffix the format type. Ex: http://localhost:8080/geoPoints?type=json
37
47
  # http://localhost:8080/geoPoints/json
38
48
  # http://localhost:8080/geoPoints/csv
39
49
  # If no suffix matches, defaults to YAML
40
- config :map_url, :validate => :string, :required => true
50
+ config :url, :validate => :string
51
+
52
+ #HTTP method
53
+ config :method, :validate => :string, :default => "get"
54
+
55
+
41
56
 
42
57
  # When using a map file or url, this setting will indicate how frequently
43
58
  # (in seconds) logstash will check the YAML file or url for updates.
44
59
  config :refresh_interval, :validate => :number, :default => 300
45
60
 
46
- # The destination field you wish to populate with the mapd code. The default
47
- # is a field named `mapping`. Set this to the same value as source if you want
48
- # to do a substitution, in this case filter will allways succeed. This will clobber
49
- # the old value of the source field!
50
- config :destination, :validate => :string, :default => "mapping"
51
-
52
61
  # In case no mapping occurs in the event (no matches), this will add a default
53
62
  # mapping string, which will always populate `field`, if the match failed.
54
- #
55
- # For example, if we have configured `fallback => "no match"`, using this map:
56
- # [source,ruby]
57
- # foo: bar
58
- #
59
- # Then, if logstash received an event with the field `foo` set to `bar`, the destination
60
- # field would be set to `bar`. However, if logstash received an event with `foo` set to `nope`,
61
- # then the destination field would still be populated, but with the value of `no match`.
62
- # This configuration can be dynamic and include parts of the event using the `%{field}` syntax.
63
- config :fallback, :validate => :string
63
+ config :default_values, :validate => :array, :default => {}
64
64
 
65
65
  def get_map
66
66
  @my_map
67
67
  end
68
68
 
69
- def set_map(map)
70
- @my_map = map;
69
+ def fill_map(data)
70
+ get_map.merge!(data)
71
71
  end
72
72
 
73
73
  public
74
74
  def register
75
75
  @my_map = {}
76
76
  @next_refresh = Time.now + @refresh_interval
77
- download_ws(@map_url, true)
77
+ load(true)
78
78
  @logger.debug? and @logger.debug("#{self.class.name}: map - ", :map => get_map)
79
79
  type = 'Exact'
80
80
  @logger.debug? and @logger.debug("#{self.class.name}: map mapping method - "+type)
@@ -83,101 +83,116 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base
83
83
  # def register
84
84
 
85
85
  def json_loader(data)
86
- get_map.merge!(JSON.parse(File.read(data)))
86
+ fill_map(JSON.parse(data))
87
87
  end
88
88
 
89
89
  def csv_loader(data)
90
- data = CSV.read(data).inject(Hash.new) do |acc, v|
90
+ data = CSV.parse(data).inject(Hash.new) do |acc, v|
91
91
  acc[v[0]] = v[1]
92
92
  acc
93
93
  end
94
- get_map.merge!(data)
94
+ fill_map(data)
95
95
  end
96
96
 
97
97
  def yml_loader(data)
98
- get_map.merge!(YAML.load_file(data))
98
+ fill_map(YAML.load(data))
99
99
  end
100
100
 
101
- def load_file(registering, extension, data)
101
+ def load_data(registering, data)
102
102
  begin
103
- if extension.equal?('.json')
103
+ if @format.eql?('json')
104
104
  return json_loader(data)
105
- elsif extension.end_with?('.csv')
105
+ elsif @format.eql?('csv')
106
106
  return csv_loader(data)
107
107
  end
108
108
  yml_loader(data)
109
109
  rescue Exception => _
110
110
  if registering
111
- raise "#{self.class.name}: Bad Syntax in map file #{file_name}"
111
+ raise "#{self.class.name}: Bad Syntax in data #{data}"
112
112
  else
113
- @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => file_name)
113
+ @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => data)
114
114
  end
115
115
  end
116
116
  end
117
117
 
118
- def get_extension(path)
119
- if path.end_with?('json')
120
- return '.json'
121
- elsif path.end_with?('csv')
122
- return '.csv'
123
- end
124
- '.yml'
125
- end
126
-
127
- public
128
- def download_ws(path, registering=false)
129
- extension = get_extension(path)
130
- temp_extension = '_temp'+extension;
131
- file_name = Digest::SHA1.hexdigest path
118
+ def load(registering=false)
132
119
  begin
133
- File.open(file_name+temp_extension, 'wb') do |saved_file|
134
- open(path, 'rb') do |read_file|
135
- saved_file.write(read_file.read)
136
- end
120
+ if @type=='webservice'
121
+ route = @url
122
+ data = get_webservice_content(route)
123
+ elsif @type=='file'
124
+ route = @path
125
+ data = get_file_content(route)
137
126
  end
127
+ load_data(registering, data)
138
128
  rescue Exception => _
139
129
  if registering
140
- raise "#{self.class.name}: Failed to initialize with #{file_name} and path #{path}"
130
+ raise "#{self.class.name}: Failed to initialize with type #{type} and route #{route}"
141
131
  end
142
- @logger.warn("#{self.class.name}: Something happened with URL. Continuing with old map", :map_path => file_name, :path => path)
132
+ @logger.warn("#{self.class.name}: Something happened with load. Continuing with old map", :type => @type, :route => route)
143
133
  end
134
+ end
144
135
 
145
- begin
146
- load_file(registering, extension, file_name+temp_extension)
147
- FileUtils.mv(file_name+temp_extension, file_name+extension)
148
- rescue Exception => _
149
- FileUtils.rm_f(file_name+temp_extension)
136
+ # def load
137
+
138
+ def get_file_content(file)
139
+ data = ''
140
+ File.open(file, 'r') do |read_file|
141
+ data +=read_file.read
142
+ end
143
+ data
144
+ end
145
+
146
+ # def get_file_content
147
+
148
+ def get_webservice_content(path)
149
+ case @method
150
+ when "get"
151
+ data = RestClient.get sprint(path), sprint(@headers)
152
+ when "post"
153
+ data = RestClient.post sprint(path), sprint(@params), sprint(@headers)
150
154
  end
155
+ data
151
156
  end
152
157
 
153
- # def download_yaml
158
+ # def get_webservice_content
159
+
160
+ def sprint(hash)
161
+ hash
162
+ end
154
163
 
155
164
  public
156
165
  def filter(event)
157
166
  if @next_refresh < Time.now
158
- download_ws(@map_url)
167
+ load()
159
168
  @next_refresh = Time.now + @refresh_interval
160
169
  @logger.info('downloading and refreshing map file')
161
170
  end
162
-
163
- return unless event.include?(@field) # Skip mapping in case event does not have @event field.
164
- return if event.include?(@destination) and not @override # Skip mapping in case @destination field already exists and @override is disabled.
165
-
166
171
  begin
167
- source = event[@field].is_a?(Array) ? event[@field].first.to_s : event[@field].to_s
168
- matched = false
169
- if get_map.include?(source)
170
- event[@destination] = get_map[source]
171
- matched = true
172
- end
173
-
174
- if not matched and @fallback
175
- event[@destination] = event.sprintf(@fallback)
176
- matched = true
172
+ matched = true
173
+ if @override
174
+ result = event;
175
+ else
176
+ event[@result_key] = {}
177
+ result = event[@result_key]
177
178
  end
178
- filter_matched(event) if matched or @field == @destination
179
+ @fields.each { |key|
180
+ key_string = key.to_s
181
+ if event.include?(key_string)
182
+ val_string = event[key_string]
183
+ if get_map.include?(val_string)
184
+ result[key_string] = get_map[val_string]
185
+ else
186
+ if @default_values and @default_values.include?(key_string)
187
+ result[key_string] = event.sprintf(@default_values[key_string])
188
+ end
189
+ matched = false
190
+ end
191
+ end
192
+ }
193
+ filter_matched(event) if matched and result.length == @fields.length
179
194
  rescue Exception => e
180
- @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @field, :event => event)
195
+ @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @fields, :event => event)
181
196
  end
182
197
  end # def filter
183
198
  end # class LogStash::Filters::LookUp
@@ -1,13 +1,13 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-filter-lookup'
3
- s.version = '1.2.1'
3
+ s.version = '2.0.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
- s.summary = "A general search and replace tool which uses a configured web service to determine replacement values."
6
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-filter-lookup. This gem is not a stand-alone program. See https://github.com/angel9484/logstash-filter-lookup"
7
- s.authors = ["Elastic"]
5
+ s.summary = 'A general search and replace tool which uses a configured web service to determine replacement values.'
6
+ s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-filter-lookup. This gem is not a stand-alone program. See https://github.com/angel9484/logstash-filter-lookup'
7
+ s.authors = ['Elastic']
8
8
  s.email = 'info@elastic.co'
9
- s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
10
- s.require_paths = ["lib"]
9
+ s.homepage = 'http://www.elastic.co/guide/en/logstash/current/index.html'
10
+ s.require_paths = ['lib']
11
11
 
12
12
  # Files
13
13
  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
@@ -16,9 +16,10 @@ Gem::Specification.new do |s|
16
16
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
17
 
18
18
  # Special flag to let us know this is actually a logstash plugin
19
- s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
19
+ s.metadata = {'logstash_plugin' => 'true', 'logstash_group' => 'filter'}
20
20
 
21
21
  # Gem dependencies
22
- s.add_runtime_dependency "logstash-core", ">= 1.5.0", "< 3.0.0"
22
+ s.add_runtime_dependency 'logstash-core', '>= 1.5.0', '< 3.0.0'
23
+ s.add_runtime_dependency 'rest-client', '>= 1.8.0'
23
24
  s.add_development_dependency 'logstash-devutils', '~> 0'
24
25
  end
@@ -2,7 +2,6 @@
2
2
  require "logstash/devutils/rspec/spec_helper"
3
3
  require "logstash/filters/lookup"
4
4
  require "webmock/rspec"
5
- require 'digest/sha1'
6
5
  WebMock.disable_net_connect!(allow_localhost: true)
7
6
 
8
7
  describe LogStash::Filters::LookUp do
@@ -10,126 +9,305 @@ describe LogStash::Filters::LookUp do
10
9
  let(:config) { Hash.new }
11
10
  subject { described_class.new(config) }
12
11
 
12
+ describe "file mapping" do
13
+ config <<-CONFIG
14
+ filter {
15
+ lookup {
16
+ fields => ["status"]
17
+ type => "file"
18
+ path => "filename"
19
+ }
20
+ }
21
+ CONFIG
22
+ content = "\
23
+ '200': OK\n\
24
+ '300': Redirect\n\
25
+ '400': Client Error\n\
26
+ '500': Server Error"
27
+ filename = 'filename'
28
+
29
+ RSpec.configure do |config|
30
+ config.before(:each) do
31
+ allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
32
+ end
33
+ end
34
+ sample({"status" => "200"}) do
35
+ insist { subject["lookup_result"]["status"] } == "OK"
36
+ end
37
+ end
38
+
13
39
  describe "webserver mapping" do
14
- config <<-CONFIG
40
+ config <<-CONFIG
15
41
  filter {
16
42
  lookup {
17
- field => "status"
18
- destination => "mapping"
19
- map_url => "http://dummyurl/"
43
+ fields => ["status"]
44
+ format => "yml"
45
+ url => "http://dummyurl/"
20
46
  }
21
47
  }
22
- CONFIG
23
-
24
- RSpec.configure do |config|
25
- hash = Digest::SHA1.hexdigest 'http://dummyurl/'
26
- config.before(:each) do
27
- FileUtils.rm_rf(hash+'.yml')
28
- stub_request(:get, "http://dummyurl/").
29
- with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
30
- to_return(:status => 200, :body => "\
48
+ CONFIG
49
+
50
+ RSpec.configure do |config|
51
+ config.before(:each) do
52
+ stub_request(:any, "http://dummyurl/").
53
+ to_return(:status => 200, :body => "\
31
54
  '200': OK\n\
32
55
  '300': Redirect\n\
33
56
  '400': Client Error\n\
34
57
  '500': Server Error", :headers => {})
35
- end
36
- config.after(:all) do
37
- FileUtils.rm_rf(hash+'.yml')
38
- end
39
58
  end
59
+ end
40
60
 
41
- sample("status" => "200") do
42
- insist { subject["mapping"] } == "OK"
43
- end
61
+ sample({"status" => "200"}) do
62
+ insist { subject["lookup_result"]["status"] } == "OK"
63
+ end
44
64
  end
45
65
 
46
66
  describe "webserver mapping existing YML" do
47
- config <<-CONFIG
67
+ config <<-CONFIG
48
68
  filter {
49
69
  lookup {
50
- field => "status"
51
- destination => "mapping"
52
- map_url => "http://dummyurl/"
70
+ fields => ["status"]
71
+ format => "yml"
72
+ url => "http://dummyurl/"
53
73
  }
54
74
  }
55
- CONFIG
56
-
57
- RSpec.configure do |config|
58
- hash = Digest::SHA1.hexdigest 'http://dummyurl/'
59
- config.before(:each) do
60
- FileUtils.rm_rf(hash+'.yml')
61
- File.open(hash+'.yml', 'wb') { |f| f.write("\
62
- '200': OKF\n\
63
- '300': Redirect\n\
64
- '400': Client Error\n\
65
- '500': Server Error") }
66
- stub_request(:get, "http://dummyurl/").
67
- with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
68
- to_return(:status => 200, :body => "\
75
+ CONFIG
76
+
77
+ RSpec.configure do |config|
78
+ config.before(:each) do
79
+ stub_request(:get, "http://dummyurl/").
80
+ to_return(:status => 200, :body => "\
69
81
  '200': OK\n\
70
82
  '300': Redirect\n\
71
83
  '400': Client Error\n\
72
84
  '500': Server Error", :headers => {})
73
- end
74
- config.after(:all) do
75
- FileUtils.rm_rf(hash+'.yml')
76
- end
77
85
  end
86
+ end
78
87
 
79
- sample("status" => "200") do
80
- insist { subject["mapping"] } == "OK"
81
- end
88
+ sample({"status" => "200"}) do
89
+ insist { subject["lookup_result"]["status"] } == "OK"
90
+ end
82
91
  end
83
92
 
84
- describe "webserver mapping not valid" do
85
- config <<-CONFIG
93
+ describe "webserver mapping not valid on register" do
94
+ config <<-CONFIG
86
95
  filter {
87
96
  lookup {
88
- field => "status"
89
- destination => "mapping"
90
- map_url => "http://dummyurl/"
97
+ fields => ["status"]
98
+ format => "yml"
99
+ url => "http://dummyurl/"
91
100
  }
92
101
  }
93
- CONFIG
94
-
95
- RSpec.configure do |config|
96
- hash = Digest::SHA1.hexdigest 'http://dummyurl/'
97
- config.before(:each) do
98
- FileUtils.rm_rf(hash+'.yml')
99
- stub_request(:get, "http://dummyurl/").
100
- with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
101
- to_return(:status => 200, :body => "\
102
+ CONFIG
103
+
104
+ RSpec.configure do |config|
105
+ config.before(:each) do
106
+ stub_request(:get, "http://dummyurl/").
107
+ to_return(:status => 200, :body => "\
102
108
  '200': OK\n\
103
109
  '300': Redirect\n\
104
110
  '400', Client Error\n\
105
111
  '500': Server Error", :headers => {})
106
- end
107
- config.after(:all) do
108
- FileUtils.rm_rf(hash+'.yml')
109
- end
110
112
  end
113
+ end
114
+
115
+ sample({"status" => "200"}) do
116
+ expect { subject }.to raise_error
117
+ end
118
+ end
119
+
120
+ describe "webserver mapping JSON" do
121
+ config <<-CONFIG
122
+ filter {
123
+ lookup {
124
+ fields => ["status"]
125
+ format => "json"
126
+ url => "http://dummyurl/json"
127
+ }
128
+ }
129
+ CONFIG
130
+
131
+ RSpec.configure do |config|
132
+ config.before(:each) do
133
+ stub_request(:get, "http://dummyurl/json").
134
+ to_return(:status => 200, :body => '{
135
+ "200": "OK",
136
+ "300": "Redirect",
137
+ "400": "Client Error",
138
+ "500": "Server Error"
139
+ }', :headers => {})
140
+ end
141
+ end
142
+
143
+ sample({"status" => "200"}) do
144
+ insist { subject["lookup_result"]["status"] } == "OK"
145
+ end
146
+ end
147
+
148
+ describe "webserver mapping existing JSON" do
149
+ config <<-CONFIG
150
+ filter {
151
+ lookup {
152
+ fields => ["status"]
153
+ format => "json"
154
+ url => "http://dummyurl/json"
155
+ }
156
+ }
157
+ CONFIG
158
+
159
+ RSpec.configure do |config|
160
+ config.before(:each) do
161
+ stub_request(:get, "http://dummyurl/json").
162
+ to_return(:status => 200, :body => '{
163
+ "200": "OK",
164
+ "300": "Redirect",
165
+ "400": "Client Error",
166
+ "500": "Server Error"
167
+ }', :headers => {})
168
+ end
169
+ end
170
+
171
+ sample({"status" => "200"}) do
172
+ insist { subject["lookup_result"]["status"] } == "OK"
173
+ end
174
+ end
175
+
176
+ describe "webserver mapping not valid JSON on register" do
177
+ config <<-CONFIG
178
+ filter {
179
+ lookup {
180
+ fields => ["status"]
181
+ format => "json"
182
+ url => "http://dummyurl/json"
183
+ }
184
+ }
185
+ CONFIG
186
+
187
+ RSpec.configure do |config|
188
+ config.before(:each) do
189
+ stub_request(:get, "http://dummyurl/json").
190
+ to_return(:status => 200, :body => '{
191
+ "200": "OK",
192
+ "300": "Redirect",
193
+ "400": "Client Error",
194
+ "500", "Server Error"
195
+ }', :headers => {})
196
+ end
197
+ end
198
+
199
+ sample({"status" => "200"}) do
200
+ expect { subject }.to raise_error
201
+ end
202
+ end
111
203
 
112
- sample("status" => "200") do
113
- insist { subject["mapping"] } == nil
204
+ describe "webserver mapping CSV" do
205
+ config <<-CONFIG
206
+ filter {
207
+ lookup {
208
+ fields => ["status"]
209
+ format => "csv"
210
+ url => "http://dummyurl/csv"
211
+ }
212
+ }
213
+ CONFIG
214
+
215
+ RSpec.configure do |config|
216
+ config.before(:each) do
217
+ stub_request(:get, "http://dummyurl/csv").
218
+ to_return(:status => 200, :body => "200,OK
219
+ 300,Redirect
220
+ 400,Client Error
221
+ 500,Server Error
222
+ ", :headers => {})
114
223
  end
224
+ end
225
+
226
+ sample({"status" => "200"}) do
227
+ insist { subject["lookup_result"]["status"] } == "OK"
228
+ end
115
229
  end
116
230
 
117
- context "allow sprintf" do
118
- let(:config) do
119
- {
120
- "field" => "status",
121
- "destination" => "mapping",
122
- "fallback" => "%{missing_mapping}",
123
- "map_url" => "http://dummyurl/"
124
- }
231
+ describe "webserver mapping existing CSV" do
232
+ config <<-CONFIG
233
+ filter {
234
+ lookup {
235
+ fields => ["status"]
236
+ format => "csv"
237
+ url => "http://dummyurl/csv"
238
+ }
239
+ }
240
+ CONFIG
241
+
242
+ RSpec.configure do |config|
243
+ config.before(:each) do
244
+ stub_request(:get, "http://dummyurl/csv").
245
+ to_return(:status => 200, :body => "200,OK
246
+ 300,Redirect
247
+ 400,Client Error
248
+ 500,Server Error
249
+ ", :headers => {})
125
250
  end
251
+ end
252
+
253
+ sample({"status" => "200"}) do
254
+ insist { subject["lookup_result"]["status"] } == "OK"
255
+ end
256
+ end
126
257
 
127
- let(:event) { LogStash::Event.new("status" => "200", "missing_mapping" => "missing no match") }
258
+ describe "webserver mapping not valid CSV" do
259
+ config <<-CONFIG
260
+ filter {
261
+ lookup {
262
+ fields => ["status"]
263
+ format => "csv"
264
+ url => "http://dummyurl/csv"
265
+ }
266
+ }
267
+ CONFIG
128
268
 
129
- it "return the exact mapping" do
130
- subject.register
131
- subject.filter(event)
132
- expect(event["mapping"]).to eq("missing no match")
269
+ RSpec.configure do |config|
270
+ config.before(:each) do
271
+ stub_request(:get, "http://dummyurl/csv").
272
+ to_return(:status => 200, :body => "200OK
273
+ 300,Redirect
274
+ 400,Client Error
275
+ 500:Server Error
276
+ ", :headers => {})
133
277
  end
134
278
  end
279
+
280
+ sample({"status" => "200"}) do
281
+ insist { subject["lookup_result"]["status"] } == nil
282
+ end
283
+ end
284
+
285
+ context "allow sprintf" do
286
+ let(:config) do
287
+ {
288
+ "fields" => ["status"],
289
+ "default_values" => {"status"=>"%{missing_mapping}"},
290
+ "type" => "file",
291
+ "path"=>"filename"
292
+ }
293
+ end
294
+ content = "\
295
+ '200': OK\n\
296
+ '300': Redirect\n\
297
+ '400': Client Error\n\
298
+ '500': Server Error"
299
+ filename = 'filename'
300
+ RSpec.configure do |config|
301
+ config.before(:each) do
302
+ allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
303
+ end
304
+ end
305
+ let(:event) { LogStash::Event.new("status" => "250", "missing_mapping" => "missing no match") }
306
+
307
+ it "return the exact mapping" do
308
+ subject.register
309
+ subject.filter(event)
310
+ expect(event["lookup_result"]["status"]).to eq("missing no match")
311
+ end
312
+ end
135
313
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-lookup
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-20 00:00:00.000000000 Z
11
+ date: 2016-03-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,20 @@ dependencies:
30
30
  - - <
31
31
  - !ruby/object:Gem::Version
32
32
  version: 3.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - '>='
37
+ - !ruby/object:Gem::Version
38
+ version: 1.8.0
39
+ name: rest-client
40
+ prerelease: false
41
+ type: :runtime
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - '>='
45
+ - !ruby/object:Gem::Version
46
+ version: 1.8.0
33
47
  - !ruby/object:Gem::Dependency
34
48
  requirement: !ruby/object:Gem::Requirement
35
49
  requirements: