logstash-filter-greynoise 0.1.7 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a71aad2c5c6984ec9021757f07fb9df22117fad154368a65916673602e76b286
-  data.tar.gz: e29ffb288550c1dae21245d3b22d00bf3149090923bacd4285054bf3aea24aa0
+  metadata.gz: 9a8f0682f3292b7e21b6aeb2200b45ad16d1e71b6f60f6c74db54e6a1a7d229d
+  data.tar.gz: 248adab329c505788c4a394a839fcaf0a63ca72b76c20a8b4407839206acf191
 SHA512:
-  metadata.gz: 5a7970993a6c48f376508e61722f4668cac2b2e614dc359a4aa1bc2903f5efde5150f7bc7ed348260f6d93da751815aea6e015f5c2c3390cea52f147430e5702
-  data.tar.gz: 773319bb84fb3857b2183177a309a2e6853f399f54228127328e262b1858ab5a2743ec734f43efdd79b5a3905ce2d947d2dcd5fcf94e2db68bc6645d70234c7a
+  metadata.gz: f10678522e0686ffd212fa53275da708daa1900bbb4e13e982e91e491f08c2815d39f3d78160a1d9150960108d4377bf9aa4e59463fd7afe4614ac0adfb6056b
+  data.tar.gz: 8d07a3c74c739141c08ce930eed271b3c0ea70aa52d4023afbcf0ee9c1970f1df2485a5a365a94958836db8aabfac63b329fbb1334122505ff668656602c7caf

data/DEVELOPER.md

@@ -1,2 +1,19 @@
 # logstash-filter-greynoise
-Example filter plugin. This should help bootstrap your effort to write your own filter plugin!
+
+You can build this project on your machine locally or alternative use the docker file to build and run an environment in Docker.
+
+To use Docker, run the following:
+
+```
+# build the docker container for local dev
+make build-docker
+
+# shell into docker environment
+make shell-docker
+
+# set GN key for tests
+export GN_API_KEY=<GN key>
+
+# run tests
+bundle exec rspec
+``` 

data/README.md

@@ -5,7 +5,7 @@ It is fully free and fully open source. The license is Apache 2.0, meaning you a
 
 ## Documentation
 
-The Greynoise filter adds information about IP addresses from logstash events via the Greynoise API.
+The GreyNoise filter adds information about IP addresses from logstash events via the GreyNoise API.
 
 GreyNoise is a system that collects and analyzes data on Internet-wide scanners.
 GreyNoise collects data on benign scanners such as Shodan.io, as well as malicious actors like SSH and telnet worms.
@@ -26,19 +26,34 @@ $LS_HOME/bin/logstash-plugin install logstash-filter-greynoise-0.1.7.gem
 ```
 
 ### 2. Filter Configuration
-Add the following inside the filter section of your logstash configuration:
 
 ```sh
 filter {
   greynoise {
-    ip => "ip_value"                 # string (required, reference to ip address field)
-    key => "your_greynoise_key"      # string (optional, no default)
-    target => "greynoise"            # string (optional, default = greynoise)
-    hit_cache_size => 100            # number (optional, default = 0)
-    hit_cache_ttl => 6               # number (optional, default = 60) 
+    ip             => "ip_value"              # string (required, reference to ip address field)
+    full_context   => true                    # bool (optional, whether to use context lookup, default false)
+    key            => "your_greynoise_key"    # string (required)
+    target         => "greynoise"             # string (optional, default = greynoise)
+    hit_cache_size => 100                     # number (optional, default = 0)
+    hit_cache_ttl  => 6                       # number (optional, default = 60) 
   }
 }
 ```
+The GreyNoise Logstash filter plugin can be used in two different modes:
+
+##### Quick IP Enrichment
+In this mode documents (default) are enriched with a bool field `seen` which is true if GreyNoise has seen this IP or false otherwise. 
+
+This mode is faster than doing full enrichment and should be used for better performance on high-volume/-throughput event streams.
+If you have a data pipeline with multiple enrichment points, you can use the boolean field to later enrich the document with IP information from GreyNoise's context endpoint.
+
+##### Full IP Enrichment
+
+In this mode (`full_context => true`), documents are enriched with full context from GreyNoise API if the IP has been observed by GreyNoise. 
+
+This mode is slower than doing quick IP enrichment and might be reasonable for low-volume/-throughput event streams. 
+
+**NOTE**: Beware that the full context document from GreyNoise API can be large and as such the cache can grow quickly in size. Set the cache size accordingly to prevent high memory consumption by the cache.
 
 Print plugin version:
 
@@ -81,3 +96,5 @@ Programming is not a required skill. Whatever you've seen about open source and
 It is more important to the community that you are able to contribute.
 
 For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+
+See also Logstash [Elastic's filter plugin development guide](https://www.elastic.co/guide/en/logstash/current/filter-new-plugin.html).

data/lib/logstash/filters/greynoise.rb

@@ -4,12 +4,17 @@ require "json"
 require "logstash/namespace"
 require "ipaddr"
 require "lru_redux"
+require 'net/http'
+require 'uri'
 
+VERSION = "1.0.0"
+
+class InvalidAPIKey < StandardError
+end
 
 # This  filter will replace the contents of the default
 # message field with whatever you specify in the configuration.
 #
-# It is only intended to be used as an .
 class LogStash::Filters::Greynoise < LogStash::Filters::Base
 
   # Setting the config_name here is required. This is how you
@@ -26,8 +31,11 @@ class LogStash::Filters::Greynoise < LogStash::Filters::Base
   # ip address to use for greynoise query
   config :ip, :validate => :string, :required => true
 
+  # whether or not to use full context endpoint
+  config :full_context, :validate => :boolean, :default => false
+
   # greynoise enterprise api key
-  config :key, :validate => :string, :default => ""
+  config :key, :validate => :string, :required => true
 
   # target top level key of hash response
   config :target, :validate => :string, :default => "greynoise"
@@ -35,6 +43,9 @@ class LogStash::Filters::Greynoise < LogStash::Filters::Base
   # tag if ip address supplied is invalid
   config :tag_on_failure, :validate => :string, :default => '_greynoise_filter_invalid_ip'
 
+  # tag if API key not valid or missing
+  config :tag_on_auth_failure, :validate => :string, :default => '_greynoise_filter_invalid_api_key'
+
   # set the size of cache for successful requests
   config :hit_cache_size, :validate => :number, :default => 0
 
@@ -47,50 +58,36 @@ class LogStash::Filters::Greynoise < LogStash::Filters::Base
     if @hit_cache_size > 0
       @hit_cache = LruRedux::TTL::ThreadSafeCache.new(@hit_cache_size, @hit_cache_ttl)
     end
-
   end
 
-  # def register
 
   private
 
-  def get_free(target_ip)
-
-    uri = URI.parse("http://api.greynoise.io/v1/query/ip")
-    request = Net::HTTP::Post.new(uri)
-    request.set_form_data(
-        "ip" => target_ip,
-    )
-
-    req_options = {
-        use_ssl: uri.scheme == "https",
-    }
-    response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http|
-      http.request(request)
-    end
-    if response.is_a?(Net::HTTPSuccess)
-      JSON.parse(response.body)
-    else
-      nil
+  def lookup_ip(target_ip, api_key, context = false)
+    endpoint = "quick/"
+    if context
+      endpoint = "context/"
     end
-  end
 
-
-  private
-
-  def get_enterprise(target_ip, api_key)
-    uri = URI.parse("https://enterprise.api.greynoise.io/v2/noise/context/" + target_ip)
+    uri = URI.parse("https://api.greynoise.io/v2/noise/" + endpoint + target_ip)
     request = Net::HTTP::Get.new(uri)
     request["Key"] = api_key
-    request["User-Agent"] = "logstash-filter-greynoise"
+    request["User-Agent"] = "logstash-filter-greynoise " + VERSION
     req_options = {
         use_ssl: uri.scheme == "https",
     }
-    response = Net::HTTP.start(uri.hostname, uri.port, req_options) do |http|
+    response = Net::HTTP.start(uri.hostname, uri.port, req_options) { |http|
       http.request(request)
-    end
+    }
+
     if response.is_a?(Net::HTTPSuccess)
-      JSON.parse(response.body)
+      result = JSON.parse(response.body)
+      unless context
+        result["seen"] = result.delete("noise")
+      end
+      result
+    elsif response.is_a?(Net::HTTPUnauthorized)
+      raise InvalidAPIKey.new
     else
       nil
     end
@@ -109,42 +106,37 @@ class LogStash::Filters::Greynoise < LogStash::Filters::Base
     if valid
       @logger.error("Invalid IP address, skipping", :ip => event.sprintf(ip), :event => event.to_hash)
       event.tag(@tag_on_failure)
-    else
-      if @hit_cache
-        result = @hit_cache[event.sprintf(ip)]
-        if result
-          event.set(@target, result)
-          filter_matched(event)
-        else
-          # check if api key exists and has len of 25 or more to prevent forbidden response
-          if @key.length >= 25
-            result = get_enterprise(event.sprintf(ip), event.sprintf(key))
-            # if no key then use alpha(free) api
-          else
-            result = get_free(event.sprintf(ip))
-          end
-          unless result.nil?
-            @hit_cache[event.sprintf(ip)] = result
-            event.set(@target, result)
-            # filter_matched should go in the last line of our successful code
-            filter_matched(event)
-          end
-        end
-      else
-        if @key.length >= 25
-          result = get_enterprise(event.sprintf(ip), event.sprintf(key))
-        else
-          result = get_free(event.sprintf(ip))
-        end
+      return
+    end
+
+    if @hit_cache
+      gn_result = @hit_cache[event.sprintf(ip)]
+
+      # use cached data
+      if gn_result
+        event.set(@target, gn_result)
+        filter_matched(event)
+        return
+      end
+    end
 
-        unless result.nil?
-          event.set(@target, result)
-          filter_matched(event)
+    # use GN API, since not found in cache
+    begin
+      gn_result = lookup_ip(event.sprintf(ip), event.sprintf(key), @full_context)
+      unless gn_result.nil?
+        if @hit_cache
+          # store in cache
+          @hit_cache[event.sprintf(ip)] = gn_result
         end
+
+        event.set(@target, gn_result)
+        # filter_matched should go in the last line of our successful code
+        filter_matched(event)
       end
+    rescue InvalidAPIKey => _
+      @logger.error("unauthorized - check API key")
+      event.tag(@tag_on_auth_failure)
     end
   end
 
-  # def filter
-end # def LogStash::Filters::Greynoise
-
+end

data/logstash-filter-greynoise.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-greynoise'
-  s.version       = '0.1.7'
+  s.version       = '1.0.0'
   s.licenses      = ['Apache-2.0']
-  s.summary = 'This greynoise filter takes contents in the ip field and returns greynoise api data (see https://greynoise.io/ for more info).'
+  s.summary = 'This greynoise filter takes contents in the IP field and returns GreyNoise API data (see https://developer.greynoise.io/ for more info).'
   s.description     = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-greynoise. This gem is not a stand-alone program'
   s.homepage = 'https://github.com/nicksherron/logstash-filter-greynoise'
   s.authors       = ['nsherron90']
@@ -18,9 +18,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_development_dependency 'logstash-devutils'
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'lru_redux', "~> 1.1.0"
-
-
 end

data/spec/filters/greynoise_spec.rb

@@ -3,33 +3,47 @@ require_relative '../spec_helper'
 require "logstash/filters/greynoise"
 
 describe LogStash::Filters::Greynoise do
-
   describe "defaults" do
-    let(:config) do <<-CONFIG
+    let(:config) do
+      api_key = ENV["GN_API_KEY"]
+      <<-CONFIG
       filter {
         greynoise {
-          ip => "ip"
+          ip => "%{ip}"          
+          key => "#{api_key}"          
         }
       }
-    CONFIG
-    # end
+      CONFIG
+    end
 
     sample("ip" => "8.8.8.8") do
       insist { subject }.include?("greynoise")
-
-      expected_fields = %w(greynoise.ip greynoise.seen)
+      expected_fields = %w(ip seen code)
       expected_fields.each do |f|
         insist { subject.get("greynoise") }.include?(f)
       end
+      insist { subject.get("greynoise").get("code").equal?("0x05") }
+    end
+
+    sample("ip" => "4.2.1.A") do
+      insist { subject.get("tags") }.include?("_greynoise_filter_invalid_ip")
+    end
+  end
+
+  describe "invalid_key" do
+    let(:config) do
+      <<-CONFIG
+      filter {
+        greynoise {
+          ip => "%{ip}"          
+          key => "BAD_KEY"          
+        }
+      }
+      CONFIG
     end
+
+    sample("ip" => "8.8.8.8") do
+      insist { subject.get("tags") }.include?("_greynoise_filter_invalid_api_key")
     end
   end
 end
-#
-#
-#     sample("message" => "some text") do
-#       expect(subject).to include("message")
-#       expect(subject.get('message')).to eq('Hello World')
-#     end
-#   end
-# end

metadata

@@ -1,49 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-greynoise
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 1.0.0
 platform: ruby
 authors:
 - nsherron90
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-05-06 00:00:00.000000000 Z
+date: 2020-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.60'
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: '2.99'
-  name: logstash-core-plugin-api
+        version: '0'
+  name: logstash-devutils
+  type: :development
   prerelease: false
-  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.60'
-    - - "<="
-      - !ruby/object:Gem::Version
-        version: '2.99'
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-devutils
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  type: :runtime
   prerelease: false
-  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -51,8 +51,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.1.0
   name: lru_redux
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -97,12 +97,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.9
+rubygems_version: 3.0.6
 signing_key:
 specification_version: 4
-summary: This greynoise filter takes contents in the ip field and returns greynoise
-  api data (see https://greynoise.io/ for more info).
+summary: This greynoise filter takes contents in the IP field and returns GreyNoise
+  API data (see https://developer.greynoise.io/ for more info).
 test_files:
 - spec/filters/greynoise_spec.rb
 - spec/spec_helper.rb