logstash-filter-greynoise 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e2c91206c1dad50237898f1b3c001bed7003262e429a6681c84d57cc71ab4f9
-  data.tar.gz: b257152bd4e2bad9192026b5c7ab8a8fb72788182c775aadb32b0a15929c7e0d
+  metadata.gz: 770fe4cfb474728588617ecb1c8c4bbf62e068befba16d9597bf2faa366f0059
+  data.tar.gz: 5b412561f650fb8b3d993f418326b634344b4a627d90461250ea822422e80b8c
 SHA512:
-  metadata.gz: aad9d6870cbb48d2233e560eb8b331d92440cd4fef1d08c2a783fd6e75a7221d694f1e91fed7282fd8916ae8d0885efa9fce65aa355b389834596e9e37882c91
-  data.tar.gz: e176f3b484088a02db77786438f420b418d3d19351aa01ef2679e05778b58522ee683cc46766e0f3b5fe8dd57db33a9e7588a427b652f028c0ee87527ca49d21
+  metadata.gz: 0b1fddfb2a5509caa5712407c0c1087fe9ebb026605ea7516d4d01d490455bb890a98c9258e59c24325e6db4aed161106cd7231912810346f45b646ac12bd4f6
+  data.tar.gz: 25ef1b38d123e1f3ffebead3879083a28ddca0af759dc2b8a57a698a7f81f4a80762a32ef2b51197806f5a85d7532bfa7e8fd4cea5525c7c602d6efbda2d9107

data/CHANGELOG.md

@@ -10,4 +10,6 @@
   - Updated key check for len
 ## 0.1.5
   - Fixed nil key error
+## 0.1.6
+  - Added LRU cache options and filter for invalid ips
  

data/CONTRIBUTORS

@@ -2,7 +2,7 @@ The following is a list of people who have contributed ideas, code, bug
 reports, or in general have helped logstash along its way.
 
 Contributors:
-* nsherron90 - nsherron90@gmail.com
+* nicksherron - nsherron90@gmail.com
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/README.md

@@ -19,10 +19,10 @@ $LS_HOME/bin/logstash-plugin install logstash-filter-greynoise
 
 Or you can build it yourself:
 ```
-git clone https://github.com/nsherron90/logstash-filter-greynoise.git
+git clone https://github.com/nicksherron/logstash-filter-greynoise.git
 bundle install
 gem build logstash-filter-greynoise.gemspec
-$LS_HOME/bin/logstash-plugin install logstash-filter-greynoise-0.1.4.gem
+$LS_HOME/bin/logstash-plugin install logstash-filter-greynoise-0.1.6.gem
 ```
 
 ### 2. Filter Configuration
@@ -34,6 +34,8 @@ filter {
     ip => "ip_value"                 # string (required, reference to ip address field)
     key => "your_greynoise_key"      # string (optional, no default)
     target => "greynoise"            # string (optional, default = greynoise)
+    hit_cache_size => 100            # number (optional, default = 0)
+    hit_cache_ttl => 6               # number (optional, default = 60) 
   }
 }
 ```

data/lib/logstash/filters/greynoise.rb

@@ -1,8 +1,10 @@
 # encoding: utf-8
-require "logstash/filters/base"
+require 'logstash/filters/base'
 require "json"
 require "logstash/namespace"
-require 'faraday'
+require "faraday"
+require "ipaddr"
+require "lru_redux"
 
 
 # This  filter will replace the contents of the default
@@ -14,48 +16,113 @@ class LogStash::Filters::Greynoise < LogStash::Filters::Base
   # Setting the config_name here is required. This is how you
   # configure this filter from your Logstash config.
   #
-   #  filter {
-   #   greynoise {
-   #     ip => "ip"
-   #   }
-   #  }
+  #  filter {
+  #   greynoise {
+  #     ip => "ip"
+  #   }
+  #  }
 
   config_name "greynoise"
 
-  # Replace the message with this value.
-
+  # ip address to use for greynoise query
   config :ip, :validate => :string, :required => true
+
+  # greynoise enterprise api key
   config :key, :validate => :string, :default => ""
+
+  # target top level key of hash response
   config :target, :validate => :string, :default => "greynoise"
 
+  # tag if ip address supplied is invalid
+  config :tag_on_failure, :validate => :string, :default => '_greynoise_filter_invalid_ip'
 
+  # set the size of cache for successful requests
+  config :hit_cache_size, :validate => :number, :default => 0
+
+  # how long to cache successful requests (in seconds)
+  config :hit_cache_ttl, :validate => :number, :default => 60
 
   public
+
   def register
-  end # def register
+    if @hit_cache_size > 0
+      @hit_cache = LruRedux::TTL::ThreadSafeCache.new(@hit_cache_size, @hit_cache_ttl)
+    end
 
-  public
-  def filter(event)
+  end # def register
 
-    # check if api key exists and has len of 25 or more to prevent forbidden response
-    if @key.length >= 25
-      url = "https://enterprise.api.greynoise.io/v2/noise/context/" + event.sprintf(ip)
-      uri = URI.parse(URI.encode(url.strip))
+  private
 
-      response = Faraday.get(uri, nil, 'User-Agent' => 'logstash-filter-greynoise', Key: event.sprintf(key))
-    # if no key then use alpha(free) api
+  def get_free(target_ip)
+    url = "https://api.greynoise.io/v1/query/ip"
+    response = Faraday.post url, {:ip => target_ip}, 'User-Agent' => 'logstash-filter-greynoise'
+    if response.success?
+      JSON.parse(response.body)
     else
-      url = "https://api.greynoise.io/v1/query/ip"
-      response = Faraday.post url, { :ip => event.sprintf(ip) }, 'User-Agent' => 'logstash-filter-greynoise'
+      nil
+    end
+  end
+
+  private
 
+  def get_enterprise(target_ip, api_key)
+    url = "https://enterprise.api.greynoise.io/v2/noise/context/" + target_ip
+    uri = URI.parse(URI.encode(url.strip))
+    response = Faraday.get(uri, nil, 'User-Agent' => 'logstash-filter-greynoise', Key: api_key)
+    if response.success?
+      JSON.parse(response.body)
+    else
+      nil
     end
+  end
 
-    result = JSON.parse(response.body)
+  public
 
-    event.set(@target, result)
-    # filter_matched should go in the last line of our successful code
-    filter_matched(event)
+  def filter(event)
+    valid = nil
+    begin
+      IPAddr.new(event.sprintf(ip))
+    rescue ArgumentError => e
+      valid = e
+    end
 
+    if valid
+      @logger.error("Invalid IP address, skipping", :ip => event.sprintf(ip), :event => event.to_hash)
+      event.tag(@tag_on_failure)
+    else
+      if @hit_cache
+        result = @hit_cache[event.sprintf(ip)]
+        if result
+          event.set(@target, result)
+          filter_matched(event)
+        else
+          # check if api key exists and has len of 25 or more to prevent forbidden response
+          if @key.length >= 25
+            result = get_enterprise(event.sprintf(ip), event.sprintf(key))
+            # if no key then use alpha(free) api
+          else
+            result = get_free(event.sprintf(ip))
+          end
+          unless result.nil?
+            @hit_cache[event.sprintf(ip)] = result
+            event.set(@target, result)
+            # filter_matched should go in the last line of our successful code
+            filter_matched(event)
+          end
+        end
+      else
+        if @key.length >= 25
+          result = get_enterprise(event.sprintf(ip), event.sprintf(key))
+        else
+          result = get_free(event.sprintf(ip))
+        end
+
+        unless result.nil?
+          event.set(@target, result)
+          filter_matched(event)
+        end
+      end
+    end
   end # def filter
-end # class LogStash::Filters::Greynoise
+end # def LogStash::Filters::Greynoise
 

data/logstash-filter-greynoise.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-greynoise'
-  s.version       = '0.1.5'
+  s.version       = '0.1.6'
   s.licenses      = ['Apache-2.0']
   s.summary = 'This greynoise filter takes contents in the ip field and returns greynoise api data (see https://greynoise.io/ for more info).'
   s.description     = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-greynoise. This gem is not a stand-alone program'
-  s.homepage = 'https://github.com/nsherron90/logstash-filter-greynoise'
+  s.homepage = 'https://github.com/nicksherron/logstash-filter-greynoise'
   s.authors       = ['nsherron90']
   s.email         = 'nsherron90@gmail.com'
   s.require_paths = ['lib']
@@ -18,8 +18,10 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_development_dependency 'logstash-devutils'
-  s.add_runtime_dependency  'faraday', '~> 0.9.2'
+  s.add_runtime_dependency  'faraday', '= 0.17.1'
+  s.add_runtime_dependency 'lru_redux', "~> 1.1.0"
+
 
 end

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-greynoise
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - nsherron90
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-14 00:00:00.000000000 Z
+date: 2020-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.99'
   name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -41,17 +47,31 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: 0.17.1
   name: faraday
   prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.17.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  name: lru_redux
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.2
+        version: 1.1.0
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-greynoise.
   This gem is not a stand-alone program
@@ -70,7 +90,7 @@ files:
 - logstash-filter-greynoise.gemspec
 - spec/filters/greynoise_spec.rb
 - spec/spec_helper.rb
-homepage: https://github.com/nsherron90/logstash-filter-greynoise
+homepage: https://github.com/nicksherron/logstash-filter-greynoise
 licenses:
 - Apache-2.0
 metadata: