logstash-filter-lookup 1.2.1 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8db20eba2cf5828897c358412d6b0d406844239e
4
- data.tar.gz: 364d5cb9dd19ae95d269b23134fe20bb7f797e9c
3
+ metadata.gz: 175c91bd50e708f7d89b7db370bac124f3770d60
4
+ data.tar.gz: fb814d76271fa06046fab307df9ad0fc169d684d
5
5
  SHA512:
6
- metadata.gz: f15abca4bda7c2dcddd29c5c5d3e0002f5720adeda71487339980ccaf55bad5229180b75021662c98fa755eb9c124e2b18261e5be5b98637a86bce60e6fe5f81
7
- data.tar.gz: 87565c30b7bcecae8add5082bae3f22be5f53c9274168d24ae5ac78ad858627db1bbe6d76e85132b4f2d93953975c5e5c47481499ed72ad75f45d3c8eaa82fdd
6
+ metadata.gz: 9385b8dd385a3fe54b2908b81e7cab60179df6d5b9e26e1ecdab131b690baf56d94847ed7e8bc8ceac45889b6d1c6382b0e7d67ab310d41506ea54c7650a805a
7
+ data.tar.gz: 063b615c61acc2d109b97496b6f094b90ff4eb037a24517939902ad0d6b75297ca0ed338965d64b83d918d80f111d8fd15e9224c60f0402c6b570e2d24f474b1
@@ -1,2 +1,5 @@
1
1
  ## 1.0
2
- Initialized logstash-filter-lookup. Formerly called logstash-filter-webservicemap
2
+ Initialized logstash-filter-lookup. Formerly called logstash-filter-webservicemap
3
+
4
+ ## 2.0
5
+ Now can load a file. Uses RestClient
data/README.md CHANGED
@@ -1,8 +1,7 @@
1
- # Logstash Plugin
1
+ # logstash-filter-lookup
2
2
 
3
- This is a plugin for [Logstash](https://github.com/elastic/logstash).
3
+ New lookup plugin for dynamic lookup enrichment from multiple types of sources. There are a few different filters out there today that conduct different types of lookups, which may make sense being encapsulated within one lookup filter.
4
4
 
5
- It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
6
5
 
7
6
  ## Example
8
7
  ### 1. Web Service Example
@@ -1,10 +1,10 @@
1
1
  # encoding: utf-8
2
2
  require 'logstash/filters/base'
3
3
  require 'logstash/namespace'
4
- require 'open-uri'
5
- require 'digest/sha1'
6
4
  require 'json'
7
5
  require 'csv'
6
+ require 'rest_client'
7
+
8
8
 
9
9
  # A general search and replace tool which uses a Web service with a YAML, CSV or JSON response to determine replacement values.
10
10
  #
@@ -14,10 +14,7 @@ require 'csv'
14
14
  # matches the EXACT contents of a map entry key, the field's value will be substituted
15
15
  # with the matched key's value from the map.
16
16
  #
17
- # By default, the lookup filter will replace the contents of the
18
- # maching event field (in-place). However, by using the `destination`
19
- # configuration item, you may also specify a target event field to
20
- # populate with the new mapd value.
17
+ # The lookup filter will put the contents of the maching event fields into field result.
21
18
 
22
19
  class LogStash::Filters::LookUp < LogStash::Filters::Base
23
20
  config_name "lookup"
@@ -25,56 +22,59 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base
25
22
  # match by the map filter (e.g. `message`, `host`, `response_code`).
26
23
  #
27
24
  # If this field is an array, only the first value will be used.
28
- config :field, :validate => :string, :required => true
25
+ config :fields, :validate => :array, :required => true
29
26
 
30
- # If the destination (or target) field already exists, this configuration item specifies
31
- # whether the filter should skip mapping (default) or overwrite the target field
32
- # value with the new mapping value.
27
+ # By default false, if false, then all the result will be stored into a field called lookup_result
28
+ # If true, the same field used to lookup, will be replaced.
33
29
  config :override, :validate => :boolean, :default => false
34
30
 
31
+ # Can be webservice or file
32
+ config :type, :validate => :string, :default => 'webservice'
33
+
34
+ # Path of a file
35
+ config :path, :validate => :string
36
+
37
+ config :format, :validate => :string, :default => 'yml'
38
+
39
+ config :headers, :validate => :hash, :default => { }
40
+ config :params, :validate => :hash, :default => { }
41
+
42
+ #default field where store results if override option is false
43
+ config :result_key, :validate => :string, :default => "lookup_result"
44
+
35
45
  # The full URI path of a Web service who generates an JSON, yml or CSV format response.
36
46
  # requires to append as suffix the format type. Ex: http://localhost:8080/geoPoints?type=json
37
47
  # http://localhost:8080/geoPoints/json
38
48
  # http://localhost:8080/geoPoints/csv
39
49
  # If no suffix matches, defaults to YAML
40
- config :map_url, :validate => :string, :required => true
50
+ config :url, :validate => :string
51
+
52
+ #HTTP method
53
+ config :method, :validate => :string, :default => "get"
54
+
55
+
41
56
 
42
57
  # When using a map file or url, this setting will indicate how frequently
43
58
  # (in seconds) logstash will check the YAML file or url for updates.
44
59
  config :refresh_interval, :validate => :number, :default => 300
45
60
 
46
- # The destination field you wish to populate with the mapd code. The default
47
- # is a field named `mapping`. Set this to the same value as source if you want
48
- # to do a substitution, in this case filter will allways succeed. This will clobber
49
- # the old value of the source field!
50
- config :destination, :validate => :string, :default => "mapping"
51
-
52
61
  # In case no mapping occurs in the event (no matches), this will add a default
53
62
  # mapping string, which will always populate `field`, if the match failed.
54
- #
55
- # For example, if we have configured `fallback => "no match"`, using this map:
56
- # [source,ruby]
57
- # foo: bar
58
- #
59
- # Then, if logstash received an event with the field `foo` set to `bar`, the destination
60
- # field would be set to `bar`. However, if logstash received an event with `foo` set to `nope`,
61
- # then the destination field would still be populated, but with the value of `no match`.
62
- # This configuration can be dynamic and include parts of the event using the `%{field}` syntax.
63
- config :fallback, :validate => :string
63
+ config :default_values, :validate => :array, :default => {}
64
64
 
65
65
  def get_map
66
66
  @my_map
67
67
  end
68
68
 
69
- def set_map(map)
70
- @my_map = map;
69
+ def fill_map(data)
70
+ get_map.merge!(data)
71
71
  end
72
72
 
73
73
  public
74
74
  def register
75
75
  @my_map = {}
76
76
  @next_refresh = Time.now + @refresh_interval
77
- download_ws(@map_url, true)
77
+ load(true)
78
78
  @logger.debug? and @logger.debug("#{self.class.name}: map - ", :map => get_map)
79
79
  type = 'Exact'
80
80
  @logger.debug? and @logger.debug("#{self.class.name}: map mapping method - "+type)
@@ -83,101 +83,116 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base
83
83
  # def register
84
84
 
85
85
  def json_loader(data)
86
- get_map.merge!(JSON.parse(File.read(data)))
86
+ fill_map(JSON.parse(data))
87
87
  end
88
88
 
89
89
  def csv_loader(data)
90
- data = CSV.read(data).inject(Hash.new) do |acc, v|
90
+ data = CSV.parse(data).inject(Hash.new) do |acc, v|
91
91
  acc[v[0]] = v[1]
92
92
  acc
93
93
  end
94
- get_map.merge!(data)
94
+ fill_map(data)
95
95
  end
96
96
 
97
97
  def yml_loader(data)
98
- get_map.merge!(YAML.load_file(data))
98
+ fill_map(YAML.load(data))
99
99
  end
100
100
 
101
- def load_file(registering, extension, data)
101
+ def load_data(registering, data)
102
102
  begin
103
- if extension.equal?('.json')
103
+ if @format.eql?('json')
104
104
  return json_loader(data)
105
- elsif extension.end_with?('.csv')
105
+ elsif @format.eql?('csv')
106
106
  return csv_loader(data)
107
107
  end
108
108
  yml_loader(data)
109
109
  rescue Exception => _
110
110
  if registering
111
- raise "#{self.class.name}: Bad Syntax in map file #{file_name}"
111
+ raise "#{self.class.name}: Bad Syntax in data #{data}"
112
112
  else
113
- @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => file_name)
113
+ @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => data)
114
114
  end
115
115
  end
116
116
  end
117
117
 
118
- def get_extension(path)
119
- if path.end_with?('json')
120
- return '.json'
121
- elsif path.end_with?('csv')
122
- return '.csv'
123
- end
124
- '.yml'
125
- end
126
-
127
- public
128
- def download_ws(path, registering=false)
129
- extension = get_extension(path)
130
- temp_extension = '_temp'+extension;
131
- file_name = Digest::SHA1.hexdigest path
118
+ def load(registering=false)
132
119
  begin
133
- File.open(file_name+temp_extension, 'wb') do |saved_file|
134
- open(path, 'rb') do |read_file|
135
- saved_file.write(read_file.read)
136
- end
120
+ if @type=='webservice'
121
+ route = @url
122
+ data = get_webservice_content(route)
123
+ elsif @type=='file'
124
+ route = @path
125
+ data = get_file_content(route)
137
126
  end
127
+ load_data(registering, data)
138
128
  rescue Exception => _
139
129
  if registering
140
- raise "#{self.class.name}: Failed to initialize with #{file_name} and path #{path}"
130
+ raise "#{self.class.name}: Failed to initialize with type #{type} and route #{route}"
141
131
  end
142
- @logger.warn("#{self.class.name}: Something happened with URL. Continuing with old map", :map_path => file_name, :path => path)
132
+ @logger.warn("#{self.class.name}: Something happened with load. Continuing with old map", :type => @type, :route => route)
143
133
  end
134
+ end
144
135
 
145
- begin
146
- load_file(registering, extension, file_name+temp_extension)
147
- FileUtils.mv(file_name+temp_extension, file_name+extension)
148
- rescue Exception => _
149
- FileUtils.rm_f(file_name+temp_extension)
136
+ # def load
137
+
138
+ def get_file_content(file)
139
+ data = ''
140
+ File.open(file, 'r') do |read_file|
141
+ data +=read_file.read
142
+ end
143
+ data
144
+ end
145
+
146
+ # def get_file_content
147
+
148
+ def get_webservice_content(path)
149
+ case @method
150
+ when "get"
151
+ data = RestClient.get sprint(path), sprint(@headers)
152
+ when "post"
153
+ data = RestClient.post sprint(path), sprint(@params), sprint(@headers)
150
154
  end
155
+ data
151
156
  end
152
157
 
153
- # def download_yaml
158
+ # def get_webservice_content
159
+
160
+ def sprint(hash)
161
+ hash
162
+ end
154
163
 
155
164
  public
156
165
  def filter(event)
157
166
  if @next_refresh < Time.now
158
- download_ws(@map_url)
167
+ load()
159
168
  @next_refresh = Time.now + @refresh_interval
160
169
  @logger.info('downloading and refreshing map file')
161
170
  end
162
-
163
- return unless event.include?(@field) # Skip mapping in case event does not have @event field.
164
- return if event.include?(@destination) and not @override # Skip mapping in case @destination field already exists and @override is disabled.
165
-
166
171
  begin
167
- source = event[@field].is_a?(Array) ? event[@field].first.to_s : event[@field].to_s
168
- matched = false
169
- if get_map.include?(source)
170
- event[@destination] = get_map[source]
171
- matched = true
172
- end
173
-
174
- if not matched and @fallback
175
- event[@destination] = event.sprintf(@fallback)
176
- matched = true
172
+ matched = true
173
+ if @override
174
+ result = event;
175
+ else
176
+ event[@result_key] = {}
177
+ result = event[@result_key]
177
178
  end
178
- filter_matched(event) if matched or @field == @destination
179
+ @fields.each { |key|
180
+ key_string = key.to_s
181
+ if event.include?(key_string)
182
+ val_string = event[key_string]
183
+ if get_map.include?(val_string)
184
+ result[key_string] = get_map[val_string]
185
+ else
186
+ if @default_values and @default_values.include?(key_string)
187
+ result[key_string] = event.sprintf(@default_values[key_string])
188
+ end
189
+ matched = false
190
+ end
191
+ end
192
+ }
193
+ filter_matched(event) if matched and result.length == @fields.length
179
194
  rescue Exception => e
180
- @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @field, :event => event)
195
+ @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @fields, :event => event)
181
196
  end
182
197
  end # def filter
183
198
  end # class LogStash::Filters::LookUp
@@ -1,13 +1,13 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-filter-lookup'
3
- s.version = '1.2.1'
3
+ s.version = '2.0.0'
4
4
  s.licenses = ['Apache License (2.0)']
5
- s.summary = "A general search and replace tool which uses a configured web service to determine replacement values."
6
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-filter-lookup. This gem is not a stand-alone program. See https://github.com/angel9484/logstash-filter-lookup"
7
- s.authors = ["Elastic"]
5
+ s.summary = 'A general search and replace tool which uses a configured web service to determine replacement values.'
6
+ s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-filter-lookup. This gem is not a stand-alone program. See https://github.com/angel9484/logstash-filter-lookup'
7
+ s.authors = ['Elastic']
8
8
  s.email = 'info@elastic.co'
9
- s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
10
- s.require_paths = ["lib"]
9
+ s.homepage = 'http://www.elastic.co/guide/en/logstash/current/index.html'
10
+ s.require_paths = ['lib']
11
11
 
12
12
  # Files
13
13
  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
@@ -16,9 +16,10 @@ Gem::Specification.new do |s|
16
16
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
17
 
18
18
  # Special flag to let us know this is actually a logstash plugin
19
- s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
19
+ s.metadata = {'logstash_plugin' => 'true', 'logstash_group' => 'filter'}
20
20
 
21
21
  # Gem dependencies
22
- s.add_runtime_dependency "logstash-core", ">= 1.5.0", "< 3.0.0"
22
+ s.add_runtime_dependency 'logstash-core', '>= 1.5.0', '< 3.0.0'
23
+ s.add_runtime_dependency 'rest-client', '>= 1.8.0'
23
24
  s.add_development_dependency 'logstash-devutils', '~> 0'
24
25
  end
@@ -2,7 +2,6 @@
2
2
  require "logstash/devutils/rspec/spec_helper"
3
3
  require "logstash/filters/lookup"
4
4
  require "webmock/rspec"
5
- require 'digest/sha1'
6
5
  WebMock.disable_net_connect!(allow_localhost: true)
7
6
 
8
7
  describe LogStash::Filters::LookUp do
@@ -10,126 +9,305 @@ describe LogStash::Filters::LookUp do
10
9
  let(:config) { Hash.new }
11
10
  subject { described_class.new(config) }
12
11
 
12
+ describe "file mapping" do
13
+ config <<-CONFIG
14
+ filter {
15
+ lookup {
16
+ fields => ["status"]
17
+ type => "file"
18
+ path => "filename"
19
+ }
20
+ }
21
+ CONFIG
22
+ content = "\
23
+ '200': OK\n\
24
+ '300': Redirect\n\
25
+ '400': Client Error\n\
26
+ '500': Server Error"
27
+ filename = 'filename'
28
+
29
+ RSpec.configure do |config|
30
+ config.before(:each) do
31
+ allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
32
+ end
33
+ end
34
+ sample({"status" => "200"}) do
35
+ insist { subject["lookup_result"]["status"] } == "OK"
36
+ end
37
+ end
38
+
13
39
  describe "webserver mapping" do
14
- config <<-CONFIG
40
+ config <<-CONFIG
15
41
  filter {
16
42
  lookup {
17
- field => "status"
18
- destination => "mapping"
19
- map_url => "http://dummyurl/"
43
+ fields => ["status"]
44
+ format => "yml"
45
+ url => "http://dummyurl/"
20
46
  }
21
47
  }
22
- CONFIG
23
-
24
- RSpec.configure do |config|
25
- hash = Digest::SHA1.hexdigest 'http://dummyurl/'
26
- config.before(:each) do
27
- FileUtils.rm_rf(hash+'.yml')
28
- stub_request(:get, "http://dummyurl/").
29
- with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
30
- to_return(:status => 200, :body => "\
48
+ CONFIG
49
+
50
+ RSpec.configure do |config|
51
+ config.before(:each) do
52
+ stub_request(:any, "http://dummyurl/").
53
+ to_return(:status => 200, :body => "\
31
54
  '200': OK\n\
32
55
  '300': Redirect\n\
33
56
  '400': Client Error\n\
34
57
  '500': Server Error", :headers => {})
35
- end
36
- config.after(:all) do
37
- FileUtils.rm_rf(hash+'.yml')
38
- end
39
58
  end
59
+ end
40
60
 
41
- sample("status" => "200") do
42
- insist { subject["mapping"] } == "OK"
43
- end
61
+ sample({"status" => "200"}) do
62
+ insist { subject["lookup_result"]["status"] } == "OK"
63
+ end
44
64
  end
45
65
 
46
66
  describe "webserver mapping existing YML" do
47
- config <<-CONFIG
67
+ config <<-CONFIG
48
68
  filter {
49
69
  lookup {
50
- field => "status"
51
- destination => "mapping"
52
- map_url => "http://dummyurl/"
70
+ fields => ["status"]
71
+ format => "yml"
72
+ url => "http://dummyurl/"
53
73
  }
54
74
  }
55
- CONFIG
56
-
57
- RSpec.configure do |config|
58
- hash = Digest::SHA1.hexdigest 'http://dummyurl/'
59
- config.before(:each) do
60
- FileUtils.rm_rf(hash+'.yml')
61
- File.open(hash+'.yml', 'wb') { |f| f.write("\
62
- '200': OKF\n\
63
- '300': Redirect\n\
64
- '400': Client Error\n\
65
- '500': Server Error") }
66
- stub_request(:get, "http://dummyurl/").
67
- with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
68
- to_return(:status => 200, :body => "\
75
+ CONFIG
76
+
77
+ RSpec.configure do |config|
78
+ config.before(:each) do
79
+ stub_request(:get, "http://dummyurl/").
80
+ to_return(:status => 200, :body => "\
69
81
  '200': OK\n\
70
82
  '300': Redirect\n\
71
83
  '400': Client Error\n\
72
84
  '500': Server Error", :headers => {})
73
- end
74
- config.after(:all) do
75
- FileUtils.rm_rf(hash+'.yml')
76
- end
77
85
  end
86
+ end
78
87
 
79
- sample("status" => "200") do
80
- insist { subject["mapping"] } == "OK"
81
- end
88
+ sample({"status" => "200"}) do
89
+ insist { subject["lookup_result"]["status"] } == "OK"
90
+ end
82
91
  end
83
92
 
84
- describe "webserver mapping not valid" do
85
- config <<-CONFIG
93
+ describe "webserver mapping not valid on register" do
94
+ config <<-CONFIG
86
95
  filter {
87
96
  lookup {
88
- field => "status"
89
- destination => "mapping"
90
- map_url => "http://dummyurl/"
97
+ fields => ["status"]
98
+ format => "yml"
99
+ url => "http://dummyurl/"
91
100
  }
92
101
  }
93
- CONFIG
94
-
95
- RSpec.configure do |config|
96
- hash = Digest::SHA1.hexdigest 'http://dummyurl/'
97
- config.before(:each) do
98
- FileUtils.rm_rf(hash+'.yml')
99
- stub_request(:get, "http://dummyurl/").
100
- with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
101
- to_return(:status => 200, :body => "\
102
+ CONFIG
103
+
104
+ RSpec.configure do |config|
105
+ config.before(:each) do
106
+ stub_request(:get, "http://dummyurl/").
107
+ to_return(:status => 200, :body => "\
102
108
  '200': OK\n\
103
109
  '300': Redirect\n\
104
110
  '400', Client Error\n\
105
111
  '500': Server Error", :headers => {})
106
- end
107
- config.after(:all) do
108
- FileUtils.rm_rf(hash+'.yml')
109
- end
110
112
  end
113
+ end
114
+
115
+ sample({"status" => "200"}) do
116
+ expect { subject }.to raise_error
117
+ end
118
+ end
119
+
120
+ describe "webserver mapping JSON" do
121
+ config <<-CONFIG
122
+ filter {
123
+ lookup {
124
+ fields => ["status"]
125
+ format => "json"
126
+ url => "http://dummyurl/json"
127
+ }
128
+ }
129
+ CONFIG
130
+
131
+ RSpec.configure do |config|
132
+ config.before(:each) do
133
+ stub_request(:get, "http://dummyurl/json").
134
+ to_return(:status => 200, :body => '{
135
+ "200": "OK",
136
+ "300": "Redirect",
137
+ "400": "Client Error",
138
+ "500": "Server Error"
139
+ }', :headers => {})
140
+ end
141
+ end
142
+
143
+ sample({"status" => "200"}) do
144
+ insist { subject["lookup_result"]["status"] } == "OK"
145
+ end
146
+ end
147
+
148
+ describe "webserver mapping existing JSON" do
149
+ config <<-CONFIG
150
+ filter {
151
+ lookup {
152
+ fields => ["status"]
153
+ format => "json"
154
+ url => "http://dummyurl/json"
155
+ }
156
+ }
157
+ CONFIG
158
+
159
+ RSpec.configure do |config|
160
+ config.before(:each) do
161
+ stub_request(:get, "http://dummyurl/json").
162
+ to_return(:status => 200, :body => '{
163
+ "200": "OK",
164
+ "300": "Redirect",
165
+ "400": "Client Error",
166
+ "500": "Server Error"
167
+ }', :headers => {})
168
+ end
169
+ end
170
+
171
+ sample({"status" => "200"}) do
172
+ insist { subject["lookup_result"]["status"] } == "OK"
173
+ end
174
+ end
175
+
176
+ describe "webserver mapping not valid JSON on register" do
177
+ config <<-CONFIG
178
+ filter {
179
+ lookup {
180
+ fields => ["status"]
181
+ format => "json"
182
+ url => "http://dummyurl/json"
183
+ }
184
+ }
185
+ CONFIG
186
+
187
+ RSpec.configure do |config|
188
+ config.before(:each) do
189
+ stub_request(:get, "http://dummyurl/json").
190
+ to_return(:status => 200, :body => '{
191
+ "200": "OK",
192
+ "300": "Redirect",
193
+ "400": "Client Error",
194
+ "500", "Server Error"
195
+ }', :headers => {})
196
+ end
197
+ end
198
+
199
+ sample({"status" => "200"}) do
200
+ expect { subject }.to raise_error
201
+ end
202
+ end
111
203
 
112
- sample("status" => "200") do
113
- insist { subject["mapping"] } == nil
204
+ describe "webserver mapping CSV" do
205
+ config <<-CONFIG
206
+ filter {
207
+ lookup {
208
+ fields => ["status"]
209
+ format => "csv"
210
+ url => "http://dummyurl/csv"
211
+ }
212
+ }
213
+ CONFIG
214
+
215
+ RSpec.configure do |config|
216
+ config.before(:each) do
217
+ stub_request(:get, "http://dummyurl/csv").
218
+ to_return(:status => 200, :body => "200,OK
219
+ 300,Redirect
220
+ 400,Client Error
221
+ 500,Server Error
222
+ ", :headers => {})
114
223
  end
224
+ end
225
+
226
+ sample({"status" => "200"}) do
227
+ insist { subject["lookup_result"]["status"] } == "OK"
228
+ end
115
229
  end
116
230
 
117
- context "allow sprintf" do
118
- let(:config) do
119
- {
120
- "field" => "status",
121
- "destination" => "mapping",
122
- "fallback" => "%{missing_mapping}",
123
- "map_url" => "http://dummyurl/"
124
- }
231
+ describe "webserver mapping existing CSV" do
232
+ config <<-CONFIG
233
+ filter {
234
+ lookup {
235
+ fields => ["status"]
236
+ format => "csv"
237
+ url => "http://dummyurl/csv"
238
+ }
239
+ }
240
+ CONFIG
241
+
242
+ RSpec.configure do |config|
243
+ config.before(:each) do
244
+ stub_request(:get, "http://dummyurl/csv").
245
+ to_return(:status => 200, :body => "200,OK
246
+ 300,Redirect
247
+ 400,Client Error
248
+ 500,Server Error
249
+ ", :headers => {})
125
250
  end
251
+ end
252
+
253
+ sample({"status" => "200"}) do
254
+ insist { subject["lookup_result"]["status"] } == "OK"
255
+ end
256
+ end
126
257
 
127
- let(:event) { LogStash::Event.new("status" => "200", "missing_mapping" => "missing no match") }
258
+ describe "webserver mapping not valid CSV" do
259
+ config <<-CONFIG
260
+ filter {
261
+ lookup {
262
+ fields => ["status"]
263
+ format => "csv"
264
+ url => "http://dummyurl/csv"
265
+ }
266
+ }
267
+ CONFIG
128
268
 
129
- it "return the exact mapping" do
130
- subject.register
131
- subject.filter(event)
132
- expect(event["mapping"]).to eq("missing no match")
269
+ RSpec.configure do |config|
270
+ config.before(:each) do
271
+ stub_request(:get, "http://dummyurl/csv").
272
+ to_return(:status => 200, :body => "200OK
273
+ 300,Redirect
274
+ 400,Client Error
275
+ 500:Server Error
276
+ ", :headers => {})
133
277
  end
134
278
  end
279
+
280
+ sample({"status" => "200"}) do
281
+ insist { subject["lookup_result"]["status"] } == nil
282
+ end
283
+ end
284
+
285
+ context "allow sprintf" do
286
+ let(:config) do
287
+ {
288
+ "fields" => ["status"],
289
+ "default_values" => {"status"=>"%{missing_mapping}"},
290
+ "type" => "file",
291
+ "path"=>"filename"
292
+ }
293
+ end
294
+ content = "\
295
+ '200': OK\n\
296
+ '300': Redirect\n\
297
+ '400': Client Error\n\
298
+ '500': Server Error"
299
+ filename = 'filename'
300
+ RSpec.configure do |config|
301
+ config.before(:each) do
302
+ allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
303
+ end
304
+ end
305
+ let(:event) { LogStash::Event.new("status" => "250", "missing_mapping" => "missing no match") }
306
+
307
+ it "return the exact mapping" do
308
+ subject.register
309
+ subject.filter(event)
310
+ expect(event["lookup_result"]["status"]).to eq("missing no match")
311
+ end
312
+ end
135
313
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-lookup
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 2.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-20 00:00:00.000000000 Z
11
+ date: 2016-03-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,20 @@ dependencies:
30
30
  - - <
31
31
  - !ruby/object:Gem::Version
32
32
  version: 3.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - '>='
37
+ - !ruby/object:Gem::Version
38
+ version: 1.8.0
39
+ name: rest-client
40
+ prerelease: false
41
+ type: :runtime
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - '>='
45
+ - !ruby/object:Gem::Version
46
+ version: 1.8.0
33
47
  - !ruby/object:Gem::Dependency
34
48
  requirement: !ruby/object:Gem::Requirement
35
49
  requirements: