logstash-filter-lookup 1.2.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/README.md +2 -3
- data/lib/logstash/filters/lookup.rb +97 -82
- data/logstash-filter-lookup.gemspec +9 -8
- data/spec/filters/lookup_spec.rb +257 -79
- metadata +16 -2
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA1:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: 175c91bd50e708f7d89b7db370bac124f3770d60
         | 
| 4 | 
            +
              data.tar.gz: fb814d76271fa06046fab307df9ad0fc169d684d
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 9385b8dd385a3fe54b2908b81e7cab60179df6d5b9e26e1ecdab131b690baf56d94847ed7e8bc8ceac45889b6d1c6382b0e7d67ab310d41506ea54c7650a805a
         | 
| 7 | 
            +
              data.tar.gz: 063b615c61acc2d109b97496b6f094b90ff4eb037a24517939902ad0d6b75297ca0ed338965d64b83d918d80f111d8fd15e9224c60f0402c6b570e2d24f474b1
         | 
    
        data/CHANGELOG.md
    CHANGED
    
    
    
        data/README.md
    CHANGED
    
    | @@ -1,8 +1,7 @@ | |
| 1 | 
            -
            #  | 
| 1 | 
            +
            # logstash-filter-lookup
         | 
| 2 2 |  | 
| 3 | 
            -
             | 
| 3 | 
            +
            New lookup plugin for dynamic lookup enrichment from multiple types of sources. There are a few different filters out there today that conduct different types of lookups, which may make sense being encapsulated within one lookup filter.
         | 
| 4 4 |  | 
| 5 | 
            -
            It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
         | 
| 6 5 |  | 
| 7 6 | 
             
            ## Example
         | 
| 8 7 | 
             
            ### 1. Web Service Example
         | 
| @@ -1,10 +1,10 @@ | |
| 1 1 | 
             
            # encoding: utf-8
         | 
| 2 2 | 
             
            require 'logstash/filters/base'
         | 
| 3 3 | 
             
            require 'logstash/namespace'
         | 
| 4 | 
            -
            require 'open-uri'
         | 
| 5 | 
            -
            require 'digest/sha1'
         | 
| 6 4 | 
             
            require 'json'
         | 
| 7 5 | 
             
            require 'csv'
         | 
| 6 | 
            +
            require 'rest_client'
         | 
| 7 | 
            +
             | 
| 8 8 |  | 
| 9 9 | 
             
            # A general search and replace tool which uses a Web service with a YAML, CSV or JSON response to determine replacement values.
         | 
| 10 10 | 
             
            #
         | 
| @@ -14,10 +14,7 @@ require 'csv' | |
| 14 14 | 
             
            # matches the EXACT contents of a map entry key, the field's value will be substituted
         | 
| 15 15 | 
             
            # with the matched key's value from the map.
         | 
| 16 16 | 
             
            #
         | 
| 17 | 
            -
            #  | 
| 18 | 
            -
            # maching event field (in-place). However, by using the `destination`
         | 
| 19 | 
            -
            # configuration item, you may also specify a target event field to
         | 
| 20 | 
            -
            # populate with the new mapd value.
         | 
| 17 | 
            +
            # The lookup filter will put the contents of the maching event fields into field result.
         | 
| 21 18 |  | 
| 22 19 | 
             
            class LogStash::Filters::LookUp < LogStash::Filters::Base
         | 
| 23 20 | 
             
              config_name "lookup"
         | 
| @@ -25,56 +22,59 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base | |
| 25 22 | 
             
              # match by the map filter (e.g. `message`, `host`, `response_code`). 
         | 
| 26 23 | 
             
              # 
         | 
| 27 24 | 
             
              # If this field is an array, only the first value will be used.
         | 
| 28 | 
            -
              config : | 
| 25 | 
            +
              config :fields, :validate => :array, :required => true
         | 
| 29 26 |  | 
| 30 | 
            -
              #  | 
| 31 | 
            -
              #  | 
| 32 | 
            -
              # value with the new mapping value.
         | 
| 27 | 
            +
              # By default false, if false, then all the result will be stored into a field called lookup_result
         | 
| 28 | 
            +
              # If true, the same field used to lookup, will be replaced.
         | 
| 33 29 | 
             
              config :override, :validate => :boolean, :default => false
         | 
| 34 30 |  | 
| 31 | 
            +
              # Can be webservice or file
         | 
| 32 | 
            +
              config :type, :validate => :string, :default => 'webservice'
         | 
| 33 | 
            +
             | 
| 34 | 
            +
              # Path of a file
         | 
| 35 | 
            +
              config :path, :validate => :string
         | 
| 36 | 
            +
             | 
| 37 | 
            +
              config :format, :validate => :string, :default => 'yml'
         | 
| 38 | 
            +
             | 
| 39 | 
            +
              config :headers, :validate => :hash, :default => {  }
         | 
| 40 | 
            +
              config :params, :validate => :hash, :default => {  }
         | 
| 41 | 
            +
             | 
| 42 | 
            +
              #default field where store results if override option is false
         | 
| 43 | 
            +
              config :result_key, :validate => :string, :default => "lookup_result"
         | 
| 44 | 
            +
             | 
| 35 45 | 
             
              # The full URI path of a Web service who generates an JSON, yml or CSV format response.
         | 
| 36 46 | 
             
              # requires to append as suffix the format type. Ex: http://localhost:8080/geoPoints?type=json
         | 
| 37 47 | 
             
              # http://localhost:8080/geoPoints/json
         | 
| 38 48 | 
             
              # http://localhost:8080/geoPoints/csv
         | 
| 39 49 | 
             
              # If no suffix matches, defaults to YAML
         | 
| 40 | 
            -
              config : | 
| 50 | 
            +
              config :url, :validate => :string
         | 
| 51 | 
            +
             | 
| 52 | 
            +
              #HTTP method
         | 
| 53 | 
            +
              config :method, :validate => :string, :default => "get"
         | 
| 54 | 
            +
             | 
| 55 | 
            +
             | 
| 41 56 |  | 
| 42 57 | 
             
              # When using a map file or url, this setting will indicate how frequently
         | 
| 43 58 | 
             
              # (in seconds) logstash will check the YAML file or url for updates.
         | 
| 44 59 | 
             
              config :refresh_interval, :validate => :number, :default => 300
         | 
| 45 60 |  | 
| 46 | 
            -
              # The destination field you wish to populate with the mapd code. The default
         | 
| 47 | 
            -
              # is a field named `mapping`. Set this to the same value as source if you want
         | 
| 48 | 
            -
              # to do a substitution, in this case filter will allways succeed. This will clobber
         | 
| 49 | 
            -
              # the old value of the source field! 
         | 
| 50 | 
            -
              config :destination, :validate => :string, :default => "mapping"
         | 
| 51 | 
            -
             | 
| 52 61 | 
             
              # In case no mapping occurs in the event (no matches), this will add a default
         | 
| 53 62 | 
             
              # mapping string, which will always populate `field`, if the match failed.
         | 
| 54 | 
            -
               | 
| 55 | 
            -
              # For example, if we have configured `fallback => "no match"`, using this map:
         | 
| 56 | 
            -
              # [source,ruby]
         | 
| 57 | 
            -
              #     foo: bar
         | 
| 58 | 
            -
              #
         | 
| 59 | 
            -
              # Then, if logstash received an event with the field `foo` set to `bar`, the destination
         | 
| 60 | 
            -
              # field would be set to `bar`. However, if logstash received an event with `foo` set to `nope`,
         | 
| 61 | 
            -
              # then the destination field would still be populated, but with the value of `no match`.
         | 
| 62 | 
            -
              # This configuration can be dynamic and include parts of the event using the `%{field}` syntax.
         | 
| 63 | 
            -
              config :fallback, :validate => :string
         | 
| 63 | 
            +
              config :default_values, :validate => :array, :default => {}
         | 
| 64 64 |  | 
| 65 65 | 
             
              def get_map
         | 
| 66 66 | 
             
                @my_map
         | 
| 67 67 | 
             
              end
         | 
| 68 68 |  | 
| 69 | 
            -
              def  | 
| 70 | 
            -
                 | 
| 69 | 
            +
              def fill_map(data)
         | 
| 70 | 
            +
                get_map.merge!(data)
         | 
| 71 71 | 
             
              end
         | 
| 72 72 |  | 
| 73 73 | 
             
              public
         | 
| 74 74 | 
             
              def register
         | 
| 75 75 | 
             
                @my_map = {}
         | 
| 76 76 | 
             
                @next_refresh = Time.now + @refresh_interval
         | 
| 77 | 
            -
                 | 
| 77 | 
            +
                load(true)
         | 
| 78 78 | 
             
                @logger.debug? and @logger.debug("#{self.class.name}: map - ", :map => get_map)
         | 
| 79 79 | 
             
                type = 'Exact'
         | 
| 80 80 | 
             
                @logger.debug? and @logger.debug("#{self.class.name}: map mapping method - "+type)
         | 
| @@ -83,101 +83,116 @@ class LogStash::Filters::LookUp < LogStash::Filters::Base | |
| 83 83 | 
             
              # def register
         | 
| 84 84 |  | 
| 85 85 | 
             
              def json_loader(data)
         | 
| 86 | 
            -
                 | 
| 86 | 
            +
                fill_map(JSON.parse(data))
         | 
| 87 87 | 
             
              end
         | 
| 88 88 |  | 
| 89 89 | 
             
              def csv_loader(data)
         | 
| 90 | 
            -
                data = CSV. | 
| 90 | 
            +
                data = CSV.parse(data).inject(Hash.new) do |acc, v|
         | 
| 91 91 | 
             
                  acc[v[0]] = v[1]
         | 
| 92 92 | 
             
                  acc
         | 
| 93 93 | 
             
                end
         | 
| 94 | 
            -
                 | 
| 94 | 
            +
                fill_map(data)
         | 
| 95 95 | 
             
              end
         | 
| 96 96 |  | 
| 97 97 | 
             
              def yml_loader(data)
         | 
| 98 | 
            -
                 | 
| 98 | 
            +
                fill_map(YAML.load(data))
         | 
| 99 99 | 
             
              end
         | 
| 100 100 |  | 
| 101 | 
            -
              def  | 
| 101 | 
            +
              def load_data(registering, data)
         | 
| 102 102 | 
             
                begin
         | 
| 103 | 
            -
                  if  | 
| 103 | 
            +
                  if @format.eql?('json')
         | 
| 104 104 | 
             
                    return json_loader(data)
         | 
| 105 | 
            -
                  elsif  | 
| 105 | 
            +
                  elsif @format.eql?('csv')
         | 
| 106 106 | 
             
                    return csv_loader(data)
         | 
| 107 107 | 
             
                  end
         | 
| 108 108 | 
             
                  yml_loader(data)
         | 
| 109 109 | 
             
                rescue Exception => _
         | 
| 110 110 | 
             
                  if registering
         | 
| 111 | 
            -
                    raise "#{self.class.name}: Bad Syntax in  | 
| 111 | 
            +
                    raise "#{self.class.name}: Bad Syntax in data #{data}"
         | 
| 112 112 | 
             
                  else
         | 
| 113 | 
            -
                    @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path =>  | 
| 113 | 
            +
                    @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => data)
         | 
| 114 114 | 
             
                  end
         | 
| 115 115 | 
             
                end
         | 
| 116 116 | 
             
              end
         | 
| 117 117 |  | 
| 118 | 
            -
              def  | 
| 119 | 
            -
                if path.end_with?('json')
         | 
| 120 | 
            -
                  return '.json'
         | 
| 121 | 
            -
                elsif path.end_with?('csv')
         | 
| 122 | 
            -
                  return '.csv'
         | 
| 123 | 
            -
                end
         | 
| 124 | 
            -
                '.yml'
         | 
| 125 | 
            -
              end
         | 
| 126 | 
            -
             | 
| 127 | 
            -
              public
         | 
| 128 | 
            -
              def download_ws(path, registering=false)
         | 
| 129 | 
            -
                extension = get_extension(path)
         | 
| 130 | 
            -
                temp_extension = '_temp'+extension;
         | 
| 131 | 
            -
                file_name = Digest::SHA1.hexdigest path
         | 
| 118 | 
            +
              def load(registering=false)
         | 
| 132 119 | 
             
                begin
         | 
| 133 | 
            -
                   | 
| 134 | 
            -
                     | 
| 135 | 
            -
             | 
| 136 | 
            -
             | 
| 120 | 
            +
                  if @type=='webservice'
         | 
| 121 | 
            +
                    route = @url
         | 
| 122 | 
            +
                    data = get_webservice_content(route)
         | 
| 123 | 
            +
                  elsif @type=='file'
         | 
| 124 | 
            +
                    route = @path
         | 
| 125 | 
            +
                    data = get_file_content(route)
         | 
| 137 126 | 
             
                  end
         | 
| 127 | 
            +
                  load_data(registering, data)
         | 
| 138 128 | 
             
                rescue Exception => _
         | 
| 139 129 | 
             
                  if registering
         | 
| 140 | 
            -
                    raise "#{self.class.name}: Failed to initialize with #{ | 
| 130 | 
            +
                    raise "#{self.class.name}: Failed to initialize with type #{type} and route #{route}"
         | 
| 141 131 | 
             
                  end
         | 
| 142 | 
            -
                  @logger.warn("#{self.class.name}: Something happened with  | 
| 132 | 
            +
                  @logger.warn("#{self.class.name}: Something happened with load. Continuing with old map", :type => @type, :route => route)
         | 
| 143 133 | 
             
                end
         | 
| 134 | 
            +
              end
         | 
| 144 135 |  | 
| 145 | 
            -
             | 
| 146 | 
            -
             | 
| 147 | 
            -
             | 
| 148 | 
            -
                 | 
| 149 | 
            -
             | 
| 136 | 
            +
              # def load
         | 
| 137 | 
            +
             | 
| 138 | 
            +
              def get_file_content(file)
         | 
| 139 | 
            +
                data = ''
         | 
| 140 | 
            +
                File.open(file, 'r') do |read_file|
         | 
| 141 | 
            +
                  data +=read_file.read
         | 
| 142 | 
            +
                end
         | 
| 143 | 
            +
                data
         | 
| 144 | 
            +
              end
         | 
| 145 | 
            +
             | 
| 146 | 
            +
              # def get_file_content
         | 
| 147 | 
            +
             | 
| 148 | 
            +
              def get_webservice_content(path)
         | 
| 149 | 
            +
                case @method
         | 
| 150 | 
            +
                  when "get"
         | 
| 151 | 
            +
                    data = RestClient.get sprint(path), sprint(@headers)
         | 
| 152 | 
            +
                  when "post"
         | 
| 153 | 
            +
                    data = RestClient.post sprint(path), sprint(@params), sprint(@headers)
         | 
| 150 154 | 
             
                end
         | 
| 155 | 
            +
                data
         | 
| 151 156 | 
             
              end
         | 
| 152 157 |  | 
| 153 | 
            -
              # def  | 
| 158 | 
            +
              # def get_webservice_content
         | 
| 159 | 
            +
             | 
| 160 | 
            +
              def sprint(hash)
         | 
| 161 | 
            +
                hash
         | 
| 162 | 
            +
              end
         | 
| 154 163 |  | 
| 155 164 | 
             
              public
         | 
| 156 165 | 
             
              def filter(event)
         | 
| 157 166 | 
             
                if @next_refresh < Time.now
         | 
| 158 | 
            -
                   | 
| 167 | 
            +
                  load()
         | 
| 159 168 | 
             
                  @next_refresh = Time.now + @refresh_interval
         | 
| 160 169 | 
             
                  @logger.info('downloading and refreshing map file')
         | 
| 161 170 | 
             
                end
         | 
| 162 | 
            -
             | 
| 163 | 
            -
                return unless event.include?(@field) # Skip mapping in case event does not have @event field.
         | 
| 164 | 
            -
                return if event.include?(@destination) and not @override # Skip mapping in case @destination field already exists and @override is disabled.
         | 
| 165 | 
            -
             | 
| 166 171 | 
             
                begin
         | 
| 167 | 
            -
                   | 
| 168 | 
            -
                   | 
| 169 | 
            -
             | 
| 170 | 
            -
             | 
| 171 | 
            -
                     | 
| 172 | 
            -
             | 
| 173 | 
            -
             | 
| 174 | 
            -
                  if not matched and @fallback
         | 
| 175 | 
            -
                    event[@destination] = event.sprintf(@fallback)
         | 
| 176 | 
            -
                    matched = true
         | 
| 172 | 
            +
                  matched = true
         | 
| 173 | 
            +
                  if @override
         | 
| 174 | 
            +
                    result = event;
         | 
| 175 | 
            +
                  else
         | 
| 176 | 
            +
                    event[@result_key] = {}
         | 
| 177 | 
            +
                    result = event[@result_key]
         | 
| 177 178 | 
             
                  end
         | 
| 178 | 
            -
                   | 
| 179 | 
            +
                  @fields.each { |key|
         | 
| 180 | 
            +
                    key_string = key.to_s
         | 
| 181 | 
            +
                    if event.include?(key_string)
         | 
| 182 | 
            +
                      val_string = event[key_string]
         | 
| 183 | 
            +
                      if get_map.include?(val_string)
         | 
| 184 | 
            +
                        result[key_string] = get_map[val_string]
         | 
| 185 | 
            +
                      else
         | 
| 186 | 
            +
                        if @default_values and @default_values.include?(key_string)
         | 
| 187 | 
            +
                          result[key_string] = event.sprintf(@default_values[key_string])
         | 
| 188 | 
            +
                        end
         | 
| 189 | 
            +
                        matched = false
         | 
| 190 | 
            +
                      end
         | 
| 191 | 
            +
                    end
         | 
| 192 | 
            +
                  }
         | 
| 193 | 
            +
                  filter_matched(event) if matched and result.length == @fields.length
         | 
| 179 194 | 
             
                rescue Exception => e
         | 
| 180 | 
            -
                  @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @ | 
| 195 | 
            +
                  @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @fields, :event => event)
         | 
| 181 196 | 
             
                end
         | 
| 182 197 | 
             
              end # def filter
         | 
| 183 198 | 
             
            end # class LogStash::Filters::LookUp
         | 
| @@ -1,13 +1,13 @@ | |
| 1 1 | 
             
            Gem::Specification.new do |s|
         | 
| 2 2 | 
             
              s.name            = 'logstash-filter-lookup'
         | 
| 3 | 
            -
              s.version         = ' | 
| 3 | 
            +
              s.version         = '2.0.0'
         | 
| 4 4 | 
             
              s.licenses        = ['Apache License (2.0)']
         | 
| 5 | 
            -
              s.summary         =  | 
| 6 | 
            -
              s.description     =  | 
| 7 | 
            -
              s.authors         = [ | 
| 5 | 
            +
              s.summary         = 'A general search and replace tool which uses a configured web service to determine replacement values.'
         | 
| 6 | 
            +
              s.description     = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-filter-lookup. This gem is not a stand-alone program. See https://github.com/angel9484/logstash-filter-lookup'
         | 
| 7 | 
            +
              s.authors         = ['Elastic']
         | 
| 8 8 | 
             
              s.email           = 'info@elastic.co'
         | 
| 9 | 
            -
              s.homepage        =  | 
| 10 | 
            -
              s.require_paths = [ | 
| 9 | 
            +
              s.homepage        = 'http://www.elastic.co/guide/en/logstash/current/index.html'
         | 
| 10 | 
            +
              s.require_paths = ['lib']
         | 
| 11 11 |  | 
| 12 12 | 
             
              # Files
         | 
| 13 13 | 
             
              s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
         | 
| @@ -16,9 +16,10 @@ Gem::Specification.new do |s| | |
| 16 16 | 
             
              s.test_files = s.files.grep(%r{^(test|spec|features)/})
         | 
| 17 17 |  | 
| 18 18 | 
             
              # Special flag to let us know this is actually a logstash plugin
         | 
| 19 | 
            -
              s.metadata = { | 
| 19 | 
            +
              s.metadata = {'logstash_plugin' => 'true', 'logstash_group' => 'filter'}
         | 
| 20 20 |  | 
| 21 21 | 
             
              # Gem dependencies
         | 
| 22 | 
            -
              s.add_runtime_dependency  | 
| 22 | 
            +
              s.add_runtime_dependency 'logstash-core', '>= 1.5.0', '< 3.0.0'
         | 
| 23 | 
            +
              s.add_runtime_dependency 'rest-client', '>= 1.8.0'
         | 
| 23 24 | 
             
              s.add_development_dependency 'logstash-devutils', '~> 0'
         | 
| 24 25 | 
             
            end
         | 
    
        data/spec/filters/lookup_spec.rb
    CHANGED
    
    | @@ -2,7 +2,6 @@ | |
| 2 2 | 
             
            require "logstash/devutils/rspec/spec_helper"
         | 
| 3 3 | 
             
            require "logstash/filters/lookup"
         | 
| 4 4 | 
             
            require "webmock/rspec"
         | 
| 5 | 
            -
            require 'digest/sha1'
         | 
| 6 5 | 
             
            WebMock.disable_net_connect!(allow_localhost: true)
         | 
| 7 6 |  | 
| 8 7 | 
             
            describe LogStash::Filters::LookUp do
         | 
| @@ -10,126 +9,305 @@ describe LogStash::Filters::LookUp do | |
| 10 9 | 
             
              let(:config) { Hash.new }
         | 
| 11 10 | 
             
              subject { described_class.new(config) }
         | 
| 12 11 |  | 
| 12 | 
            +
              describe "file mapping" do
         | 
| 13 | 
            +
                config <<-CONFIG
         | 
| 14 | 
            +
                  filter {
         | 
| 15 | 
            +
                      lookup {
         | 
| 16 | 
            +
                          fields       => ["status"]
         | 
| 17 | 
            +
                          type => "file"
         | 
| 18 | 
            +
                          path  => "filename"
         | 
| 19 | 
            +
                      }
         | 
| 20 | 
            +
                  }
         | 
| 21 | 
            +
                CONFIG
         | 
| 22 | 
            +
                content = "\
         | 
| 23 | 
            +
                                    '200': OK\n\
         | 
| 24 | 
            +
                                    '300': Redirect\n\
         | 
| 25 | 
            +
                                    '400': Client Error\n\
         | 
| 26 | 
            +
                                    '500': Server Error"
         | 
| 27 | 
            +
                filename = 'filename'
         | 
| 28 | 
            +
             | 
| 29 | 
            +
                RSpec.configure do |config|
         | 
| 30 | 
            +
                  config.before(:each) do
         | 
| 31 | 
            +
                    allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
         | 
| 32 | 
            +
                  end
         | 
| 33 | 
            +
                end
         | 
| 34 | 
            +
                sample({"status" => "200"}) do
         | 
| 35 | 
            +
                  insist { subject["lookup_result"]["status"] } == "OK"
         | 
| 36 | 
            +
                end
         | 
| 37 | 
            +
              end
         | 
| 38 | 
            +
             | 
| 13 39 | 
             
              describe "webserver mapping" do
         | 
| 14 | 
            -
             | 
| 40 | 
            +
                config <<-CONFIG
         | 
| 15 41 | 
             
                  filter {
         | 
| 16 42 | 
             
                      lookup {
         | 
| 17 | 
            -
                           | 
| 18 | 
            -
                           | 
| 19 | 
            -
                           | 
| 43 | 
            +
                          fields       => ["status"]
         | 
| 44 | 
            +
                          format => "yml"
         | 
| 45 | 
            +
                          url  => "http://dummyurl/"
         | 
| 20 46 | 
             
                      }
         | 
| 21 47 | 
             
                  }
         | 
| 22 | 
            -
             | 
| 23 | 
            -
             | 
| 24 | 
            -
             | 
| 25 | 
            -
             | 
| 26 | 
            -
             | 
| 27 | 
            -
             | 
| 28 | 
            -
                          stub_request(:get, "http://dummyurl/").
         | 
| 29 | 
            -
                          with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
         | 
| 30 | 
            -
                          to_return(:status => 200, :body => "\
         | 
| 48 | 
            +
                CONFIG
         | 
| 49 | 
            +
             | 
| 50 | 
            +
                RSpec.configure do |config|
         | 
| 51 | 
            +
                  config.before(:each) do
         | 
| 52 | 
            +
                    stub_request(:any, "http://dummyurl/").
         | 
| 53 | 
            +
                        to_return(:status => 200, :body => "\
         | 
| 31 54 | 
             
                                    '200': OK\n\
         | 
| 32 55 | 
             
                                    '300': Redirect\n\
         | 
| 33 56 | 
             
                                    '400': Client Error\n\
         | 
| 34 57 | 
             
                                    '500': Server Error", :headers => {})
         | 
| 35 | 
            -
                      end
         | 
| 36 | 
            -
                      config.after(:all) do
         | 
| 37 | 
            -
                          FileUtils.rm_rf(hash+'.yml')
         | 
| 38 | 
            -
                      end
         | 
| 39 58 | 
             
                  end
         | 
| 59 | 
            +
                end
         | 
| 40 60 |  | 
| 41 | 
            -
             | 
| 42 | 
            -
             | 
| 43 | 
            -
             | 
| 61 | 
            +
                sample({"status" => "200"}) do
         | 
| 62 | 
            +
                  insist { subject["lookup_result"]["status"] } == "OK"
         | 
| 63 | 
            +
                end
         | 
| 44 64 | 
             
              end
         | 
| 45 65 |  | 
| 46 66 | 
             
              describe "webserver mapping existing YML" do
         | 
| 47 | 
            -
             | 
| 67 | 
            +
                config <<-CONFIG
         | 
| 48 68 | 
             
                  filter {
         | 
| 49 69 | 
             
                      lookup {
         | 
| 50 | 
            -
                           | 
| 51 | 
            -
                           | 
| 52 | 
            -
                           | 
| 70 | 
            +
                          fields       => ["status"]
         | 
| 71 | 
            +
                          format => "yml"
         | 
| 72 | 
            +
                          url  => "http://dummyurl/"
         | 
| 53 73 | 
             
                      }
         | 
| 54 74 | 
             
                  }
         | 
| 55 | 
            -
             | 
| 56 | 
            -
             | 
| 57 | 
            -
             | 
| 58 | 
            -
             | 
| 59 | 
            -
             | 
| 60 | 
            -
             | 
| 61 | 
            -
                          File.open(hash+'.yml', 'wb') { |f| f.write("\
         | 
| 62 | 
            -
                                                                   '200': OKF\n\
         | 
| 63 | 
            -
                                                                   '300': Redirect\n\
         | 
| 64 | 
            -
                                                                   '400': Client Error\n\
         | 
| 65 | 
            -
                                                                   '500': Server Error") }
         | 
| 66 | 
            -
                          stub_request(:get, "http://dummyurl/").
         | 
| 67 | 
            -
                          with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
         | 
| 68 | 
            -
                          to_return(:status => 200, :body => "\
         | 
| 75 | 
            +
                CONFIG
         | 
| 76 | 
            +
             | 
| 77 | 
            +
                RSpec.configure do |config|
         | 
| 78 | 
            +
                  config.before(:each) do
         | 
| 79 | 
            +
                    stub_request(:get, "http://dummyurl/").
         | 
| 80 | 
            +
                        to_return(:status => 200, :body => "\
         | 
| 69 81 | 
             
                                    '200': OK\n\
         | 
| 70 82 | 
             
                                    '300': Redirect\n\
         | 
| 71 83 | 
             
                                    '400': Client Error\n\
         | 
| 72 84 | 
             
                                    '500': Server Error", :headers => {})
         | 
| 73 | 
            -
                      end
         | 
| 74 | 
            -
                      config.after(:all) do
         | 
| 75 | 
            -
                          FileUtils.rm_rf(hash+'.yml')
         | 
| 76 | 
            -
                      end
         | 
| 77 85 | 
             
                  end
         | 
| 86 | 
            +
                end
         | 
| 78 87 |  | 
| 79 | 
            -
             | 
| 80 | 
            -
             | 
| 81 | 
            -
             | 
| 88 | 
            +
                sample({"status" => "200"}) do
         | 
| 89 | 
            +
                  insist { subject["lookup_result"]["status"] } == "OK"
         | 
| 90 | 
            +
                end
         | 
| 82 91 | 
             
              end
         | 
| 83 92 |  | 
| 84 | 
            -
              describe "webserver mapping not valid" do
         | 
| 85 | 
            -
             | 
| 93 | 
            +
              describe "webserver mapping not valid on register" do
         | 
| 94 | 
            +
                config <<-CONFIG
         | 
| 86 95 | 
             
                  filter {
         | 
| 87 96 | 
             
                      lookup {
         | 
| 88 | 
            -
                           | 
| 89 | 
            -
                           | 
| 90 | 
            -
                           | 
| 97 | 
            +
                          fields       => ["status"]
         | 
| 98 | 
            +
                          format => "yml"
         | 
| 99 | 
            +
                          url  => "http://dummyurl/"
         | 
| 91 100 | 
             
                      }
         | 
| 92 101 | 
             
                  }
         | 
| 93 | 
            -
             | 
| 94 | 
            -
             | 
| 95 | 
            -
             | 
| 96 | 
            -
             | 
| 97 | 
            -
             | 
| 98 | 
            -
             | 
| 99 | 
            -
                          stub_request(:get, "http://dummyurl/").
         | 
| 100 | 
            -
                          with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'Ruby'}).
         | 
| 101 | 
            -
                          to_return(:status => 200, :body => "\
         | 
| 102 | 
            +
                CONFIG
         | 
| 103 | 
            +
             | 
| 104 | 
            +
                RSpec.configure do |config|
         | 
| 105 | 
            +
                  config.before(:each) do
         | 
| 106 | 
            +
                    stub_request(:get, "http://dummyurl/").
         | 
| 107 | 
            +
                        to_return(:status => 200, :body => "\
         | 
| 102 108 | 
             
                                    '200': OK\n\
         | 
| 103 109 | 
             
                                    '300': Redirect\n\
         | 
| 104 110 | 
             
                                    '400', Client Error\n\
         | 
| 105 111 | 
             
                                    '500': Server Error", :headers => {})
         | 
| 106 | 
            -
                      end
         | 
| 107 | 
            -
                      config.after(:all) do
         | 
| 108 | 
            -
                          FileUtils.rm_rf(hash+'.yml')
         | 
| 109 | 
            -
                      end
         | 
| 110 112 | 
             
                  end
         | 
| 113 | 
            +
                end
         | 
| 114 | 
            +
             | 
| 115 | 
            +
                sample({"status" => "200"}) do
         | 
| 116 | 
            +
                  expect { subject }.to raise_error
         | 
| 117 | 
            +
                end
         | 
| 118 | 
            +
              end
         | 
| 119 | 
            +
             | 
| 120 | 
            +
              describe "webserver mapping JSON" do
         | 
| 121 | 
            +
                config <<-CONFIG
         | 
| 122 | 
            +
                  filter {
         | 
| 123 | 
            +
                      lookup {
         | 
| 124 | 
            +
                          fields       => ["status"]
         | 
| 125 | 
            +
                          format => "json"
         | 
| 126 | 
            +
                          url  => "http://dummyurl/json"
         | 
| 127 | 
            +
                      }
         | 
| 128 | 
            +
                  }
         | 
| 129 | 
            +
                CONFIG
         | 
| 130 | 
            +
             | 
| 131 | 
            +
                RSpec.configure do |config|
         | 
| 132 | 
            +
                  config.before(:each) do
         | 
| 133 | 
            +
                    stub_request(:get, "http://dummyurl/json").
         | 
| 134 | 
            +
                        to_return(:status => 200, :body => '{
         | 
| 135 | 
            +
            	"200": "OK",
         | 
| 136 | 
            +
            	"300": "Redirect",
         | 
| 137 | 
            +
            	"400": "Client Error",
         | 
| 138 | 
            +
            	"500": "Server Error"
         | 
| 139 | 
            +
            }', :headers => {})
         | 
| 140 | 
            +
                  end
         | 
| 141 | 
            +
                end
         | 
| 142 | 
            +
             | 
| 143 | 
            +
                sample({"status" => "200"}) do
         | 
| 144 | 
            +
                  insist { subject["lookup_result"]["status"] } == "OK"
         | 
| 145 | 
            +
                end
         | 
| 146 | 
            +
              end
         | 
| 147 | 
            +
             | 
| 148 | 
            +
              describe "webserver mapping existing JSON" do
         | 
| 149 | 
            +
                config <<-CONFIG
         | 
| 150 | 
            +
                  filter {
         | 
| 151 | 
            +
                      lookup {
         | 
| 152 | 
            +
                          fields       => ["status"]
         | 
| 153 | 
            +
                          format => "json"
         | 
| 154 | 
            +
                          url  => "http://dummyurl/json"
         | 
| 155 | 
            +
                      }
         | 
| 156 | 
            +
                  }
         | 
| 157 | 
            +
                CONFIG
         | 
| 158 | 
            +
             | 
| 159 | 
            +
                RSpec.configure do |config|
         | 
| 160 | 
            +
                  config.before(:each) do
         | 
| 161 | 
            +
                    stub_request(:get, "http://dummyurl/json").
         | 
| 162 | 
            +
                        to_return(:status => 200, :body => '{
         | 
| 163 | 
            +
            	"200": "OK",
         | 
| 164 | 
            +
            	"300": "Redirect",
         | 
| 165 | 
            +
            	"400": "Client Error",
         | 
| 166 | 
            +
            	"500": "Server Error"
         | 
| 167 | 
            +
            }', :headers => {})
         | 
| 168 | 
            +
                  end
         | 
| 169 | 
            +
                end
         | 
| 170 | 
            +
             | 
| 171 | 
            +
                sample({"status" => "200"}) do
         | 
| 172 | 
            +
                  insist { subject["lookup_result"]["status"] } == "OK"
         | 
| 173 | 
            +
                end
         | 
| 174 | 
            +
              end
         | 
| 175 | 
            +
             | 
| 176 | 
            +
              describe "webserver mapping not valid JSON on register" do
         | 
| 177 | 
            +
                config <<-CONFIG
         | 
| 178 | 
            +
                  filter {
         | 
| 179 | 
            +
                      lookup {
         | 
| 180 | 
            +
                          fields       => ["status"]
         | 
| 181 | 
            +
                          format => "json"
         | 
| 182 | 
            +
                          url  => "http://dummyurl/json"
         | 
| 183 | 
            +
                      }
         | 
| 184 | 
            +
                  }
         | 
| 185 | 
            +
                CONFIG
         | 
| 186 | 
            +
             | 
| 187 | 
            +
                RSpec.configure do |config|
         | 
| 188 | 
            +
                  config.before(:each) do
         | 
| 189 | 
            +
                    stub_request(:get, "http://dummyurl/json").
         | 
| 190 | 
            +
                        to_return(:status => 200, :body => '{
         | 
| 191 | 
            +
            	"200": "OK",
         | 
| 192 | 
            +
            	"300": "Redirect",
         | 
| 193 | 
            +
            	"400": "Client Error",
         | 
| 194 | 
            +
            	"500", "Server Error"
         | 
| 195 | 
            +
            }', :headers => {})
         | 
| 196 | 
            +
                  end
         | 
| 197 | 
            +
                end
         | 
| 198 | 
            +
             | 
| 199 | 
            +
                sample({"status" => "200"}) do
         | 
| 200 | 
            +
                  expect { subject }.to raise_error
         | 
| 201 | 
            +
                end
         | 
| 202 | 
            +
              end
         | 
| 111 203 |  | 
| 112 | 
            -
             | 
| 113 | 
            -
             | 
| 204 | 
            +
              describe "webserver mapping CSV" do
         | 
| 205 | 
            +
                config <<-CONFIG
         | 
| 206 | 
            +
                  filter {
         | 
| 207 | 
            +
                      lookup {
         | 
| 208 | 
            +
                          fields       => ["status"]
         | 
| 209 | 
            +
                          format => "csv"
         | 
| 210 | 
            +
                          url  => "http://dummyurl/csv"
         | 
| 211 | 
            +
                      }
         | 
| 212 | 
            +
                  }
         | 
| 213 | 
            +
                CONFIG
         | 
| 214 | 
            +
             | 
| 215 | 
            +
                RSpec.configure do |config|
         | 
| 216 | 
            +
                  config.before(:each) do
         | 
| 217 | 
            +
                    stub_request(:get, "http://dummyurl/csv").
         | 
| 218 | 
            +
                        to_return(:status => 200, :body => "200,OK
         | 
| 219 | 
            +
            300,Redirect
         | 
| 220 | 
            +
            400,Client Error
         | 
| 221 | 
            +
            500,Server Error
         | 
| 222 | 
            +
            ", :headers => {})
         | 
| 114 223 | 
             
                  end
         | 
| 224 | 
            +
                end
         | 
| 225 | 
            +
             | 
| 226 | 
            +
                sample({"status" => "200"}) do
         | 
| 227 | 
            +
                  insist { subject["lookup_result"]["status"] } == "OK"
         | 
| 228 | 
            +
                end
         | 
| 115 229 | 
             
              end
         | 
| 116 230 |  | 
| 117 | 
            -
             | 
| 118 | 
            -
             | 
| 119 | 
            -
             | 
| 120 | 
            -
                       | 
| 121 | 
            -
             | 
| 122 | 
            -
             | 
| 123 | 
            -
             | 
| 124 | 
            -
             | 
| 231 | 
            +
              describe "webserver mapping existing CSV" do
         | 
| 232 | 
            +
                config <<-CONFIG
         | 
| 233 | 
            +
                  filter {
         | 
| 234 | 
            +
                      lookup {
         | 
| 235 | 
            +
                          fields       => ["status"]
         | 
| 236 | 
            +
                          format => "csv"
         | 
| 237 | 
            +
                          url  => "http://dummyurl/csv"
         | 
| 238 | 
            +
                      }
         | 
| 239 | 
            +
                  }
         | 
| 240 | 
            +
                CONFIG
         | 
| 241 | 
            +
             | 
| 242 | 
            +
                RSpec.configure do |config|
         | 
| 243 | 
            +
                  config.before(:each) do
         | 
| 244 | 
            +
                    stub_request(:get, "http://dummyurl/csv").
         | 
| 245 | 
            +
                        to_return(:status => 200, :body => "200,OK
         | 
| 246 | 
            +
            300,Redirect
         | 
| 247 | 
            +
            400,Client Error
         | 
| 248 | 
            +
            500,Server Error
         | 
| 249 | 
            +
            ", :headers => {})
         | 
| 125 250 | 
             
                  end
         | 
| 251 | 
            +
                end
         | 
| 252 | 
            +
             | 
| 253 | 
            +
                sample({"status" => "200"}) do
         | 
| 254 | 
            +
                  insist { subject["lookup_result"]["status"] } == "OK"
         | 
| 255 | 
            +
                end
         | 
| 256 | 
            +
              end
         | 
| 126 257 |  | 
| 127 | 
            -
             | 
| 258 | 
            +
              describe "webserver mapping not valid CSV" do
         | 
| 259 | 
            +
                config <<-CONFIG
         | 
| 260 | 
            +
                  filter {
         | 
| 261 | 
            +
                      lookup {
         | 
| 262 | 
            +
                          fields       => ["status"]
         | 
| 263 | 
            +
                          format => "csv"
         | 
| 264 | 
            +
                          url  => "http://dummyurl/csv"
         | 
| 265 | 
            +
                      }
         | 
| 266 | 
            +
                  }
         | 
| 267 | 
            +
                CONFIG
         | 
| 128 268 |  | 
| 129 | 
            -
             | 
| 130 | 
            -
             | 
| 131 | 
            -
                     | 
| 132 | 
            -
             | 
| 269 | 
            +
                RSpec.configure do |config|
         | 
| 270 | 
            +
                  config.before(:each) do
         | 
| 271 | 
            +
                    stub_request(:get, "http://dummyurl/csv").
         | 
| 272 | 
            +
                        to_return(:status => 200, :body => "200OK
         | 
| 273 | 
            +
            300,Redirect
         | 
| 274 | 
            +
            400,Client Error
         | 
| 275 | 
            +
            500:Server Error
         | 
| 276 | 
            +
            ", :headers => {})
         | 
| 133 277 | 
             
                  end
         | 
| 134 278 | 
             
                end
         | 
| 279 | 
            +
             | 
| 280 | 
            +
                sample({"status" => "200"}) do
         | 
| 281 | 
            +
                  insist { subject["lookup_result"]["status"] } == nil
         | 
| 282 | 
            +
                end
         | 
| 283 | 
            +
              end
         | 
| 284 | 
            +
             | 
| 285 | 
            +
              context "allow sprintf" do
         | 
| 286 | 
            +
                let(:config) do
         | 
| 287 | 
            +
                  {
         | 
| 288 | 
            +
                      "fields" => ["status"],
         | 
| 289 | 
            +
                      "default_values" => {"status"=>"%{missing_mapping}"},
         | 
| 290 | 
            +
                      "type" => "file",
         | 
| 291 | 
            +
                      "path"=>"filename"
         | 
| 292 | 
            +
                  }
         | 
| 293 | 
            +
                end
         | 
| 294 | 
            +
                content = "\
         | 
| 295 | 
            +
                                    '200': OK\n\
         | 
| 296 | 
            +
                                    '300': Redirect\n\
         | 
| 297 | 
            +
                                    '400': Client Error\n\
         | 
| 298 | 
            +
                                    '500': Server Error"
         | 
| 299 | 
            +
                filename = 'filename'
         | 
| 300 | 
            +
                RSpec.configure do |config|
         | 
| 301 | 
            +
                  config.before(:each) do
         | 
| 302 | 
            +
                    allow(File).to receive(:open).with(filename, 'r').and_yield( StringIO.new(content) )
         | 
| 303 | 
            +
                  end
         | 
| 304 | 
            +
                end
         | 
| 305 | 
            +
                let(:event) { LogStash::Event.new("status" => "250", "missing_mapping" => "missing no match") }
         | 
| 306 | 
            +
             | 
| 307 | 
            +
                it "return the exact mapping" do
         | 
| 308 | 
            +
                  subject.register
         | 
| 309 | 
            +
                  subject.filter(event)
         | 
| 310 | 
            +
                  expect(event["lookup_result"]["status"]).to eq("missing no match")
         | 
| 311 | 
            +
                end
         | 
| 312 | 
            +
              end
         | 
| 135 313 | 
             
            end
         | 
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: logstash-filter-lookup
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version:  | 
| 4 | 
            +
              version: 2.0.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Elastic
         | 
| 8 8 | 
             
            autorequire:
         | 
| 9 9 | 
             
            bindir: bin
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2016-03- | 
| 11 | 
            +
            date: 2016-03-29 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| @@ -30,6 +30,20 @@ dependencies: | |
| 30 30 | 
             
                - - <
         | 
| 31 31 | 
             
                  - !ruby/object:Gem::Version
         | 
| 32 32 | 
             
                    version: 3.0.0
         | 
| 33 | 
            +
            - !ruby/object:Gem::Dependency
         | 
| 34 | 
            +
              requirement: !ruby/object:Gem::Requirement
         | 
| 35 | 
            +
                requirements:
         | 
| 36 | 
            +
                - - '>='
         | 
| 37 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 38 | 
            +
                    version: 1.8.0
         | 
| 39 | 
            +
              name: rest-client
         | 
| 40 | 
            +
              prerelease: false
         | 
| 41 | 
            +
              type: :runtime
         | 
| 42 | 
            +
              version_requirements: !ruby/object:Gem::Requirement
         | 
| 43 | 
            +
                requirements:
         | 
| 44 | 
            +
                - - '>='
         | 
| 45 | 
            +
                  - !ruby/object:Gem::Version
         | 
| 46 | 
            +
                    version: 1.8.0
         | 
| 33 47 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 34 48 | 
             
              requirement: !ruby/object:Gem::Requirement
         | 
| 35 49 | 
             
                requirements:
         |