logstash-filter-accesswatch 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34157f7c1c0392ebb20368b31fbf0007b79df4109c016e77a107732b4b41fbe2
-  data.tar.gz: cf687eb6c068bf62d241b838429ad1972a742e5318e41a27492b7c44d2a97078
+  metadata.gz: 2f75068c27fcfbe8f6630eeb56055d994e30fda75f0cf342fd94cb84ad2c06fc
+  data.tar.gz: 72df9db9018bd46970ac20827b17b26cc2ac11fbaee8e8348eaf57e67198888a
 SHA512:
-  metadata.gz: b280d78583d33c7768afc4ec852f3bc26da8979c1713df177cb746978d765f59ef361985d5a5e47f3cbc26088f9abd6eacdf1ca5479d7c53e8b488fafc903e68
-  data.tar.gz: c5717ab2193781ee5024f5a28f6757b330c1c631f64c11c56192e32107a5bc739efaa3e8cd7a8e6bf71b726b9a2e180d5cc7a98ebf8224dc3145f69cf05ff4fe
+  metadata.gz: 5d6831752d2352e47e7613ecf1c1edd72f82bd904fb4475e19d8ef509cc8fcbe6475fc6aef8d49be1736262cfa81dd5790cf40a24b337137caa4db3d28891553
+  data.tar.gz: 61d7c93786d2590784452dafba8833733079f72e4bbe3f40a97bd12e412dfbd208f2d7a61117bf65b86f6e1650d1565ece9dd48fd6c31227f41f1a3b56bb27a2

data/lib/logstash/filters/accesswatch.rb

@@ -1,130 +1,147 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
-require 'json'
-require 'set'
-require 'ipaddr'
-require 'interval_tree'
-require 'digest'
+require "logstash/plugin_mixins/http_client"
+require "json"
+require "digest"
+require "lru_redux"
 
 # The Access Watch filter adds information about robots visiting
 # your website based on data from our robots database.
-#
-# The following fields might be created:
-# [identity][type]      "robot"   If the visitor is a robot.
-# [reputation][status]  string    The reputation of the visitor (see below).
-# [robot][id]           number    A unique robot identifier
-# [robot][name]         string    A robot's name to display to the user.
-# [robot][url]          string    A link to the robot's page on the Access Watch database.
-#
-# Access Watch defines the following reputation statuses:
-#
-# nice	        perfect, as far as we know you can trust this entity
-# ok	          all right, so far no reason to worry about this entity
-# suspicious	  warning, nothing really bad, but the entity is on our radar
-# bad           danger, there is good reasons to watch or block this entity
-#
-# This filter requires the Access Watch `robots.json` file to run.
-#
 
 class LogStash::Filters::Accesswatch < LogStash::Filters::Base
 
+  include LogStash::PluginMixins::HttpClient
+
   config_name "accesswatch"
 
-  # The path to the Access Watch database file.
-  #
-  # If not specified, this will default to './robots.json'.
-  #
-  config :db_path, :validate => :path, :default => "./robots.json"
+  # Your API Key
+  config :api_key, :validate => :string, :required => true
+
+  # The size of the local cache, 0 to deactivate
+  config :cache_size, :validate => :number, :default => 10000
 
   # The field containing the IP address.
   config :ip_source, :validate => :string, :required => true
 
   # The field containing the User-Agent string.
-  config :ua_source, :validate => :string, :required => true
-
-  # Transform a CIDR described as a 2-array [start size]
-  # into a Ruby 3-dotted range.
-  private
-  def cidr2range(cidr)
-    first = cidr[0]
-    last = first + cidr[1]
-    (first...last)
+  config :user_agent_source, :validate => :string, :required => true
+
+  # The destination field for address data
+  config :address_destination, :validate => :string
+
+  # The destination field for user-agent data
+  config :user_agent_destination, :validate => :string
+
+  # The destination field for robot data
+  config :robot_destination, :validate => :string
+
+  # The destination field for reputation data
+  config :reputation_destination, :validate => :string
+
+  @@address_keys = ["value", "hostname", "country_code", "flags"]
+  @@robot_keys = ["id", "name", "url"]
+
+  public
+  def register
+    if @cache_size > 0
+      @cache = LruRedux::ThreadSafeCache.new(@cache_size)
+    end
   end
 
-  # Group elements of a collection by each value of a multi-valued attribute
-  private
-  def group_by_multi(coll, key)
-    res = Hash.new {|hash, key| hash[key] = Array.new}
-    coll.each {|el|
-      if !el[key].nil?
-        el[key].each {|val|
-          res[val].push(el)
-        }
-      end
+  def handle_response(response)
+    data = JSON.parse(response.body)
+    if response.code == 200
+      {:status => :success,
+       :data   => data}
+    else
+      {:status  => :error,
+       :code    => data["code"],
+       :message => data["message"]}
+    end
+  end
+
+  def url(path)
+    "http://api.access.watch#{path}"
+  end
+
+  def get_json(path)
+    response = self.client.get(self.url(path),
+                               headers: {"Api-Key"    => @api_key,
+                                         "Accept"     => "application/json",
+                                         "User-Agent" => "Access Watch Logstash Plugin/0.2.0"})
+    self.handle_response(response)
+  end
+
+  def post_json(path, data)
+    response = self.client.post(self.url(path),
+                                headers: {"Api-Key"      => @api_key,
+                                          "Accept"       => "application/json",
+                                          "Content-Type" => "application/json",
+                                          "User-Agent"   => "Access Watch Logstash Plugin/0.2.0"},
+                                body: JSON.generate(data))
+    self.handle_response(response)
+  end
+
+  def with_cache(id, &block)
+    if @cache
+      @cache.getset(id) { block.call }
+    else
+      block.call
+    end
+  end
+
+  def fetch_address(ip)
+    self.with_cache("ip-#{ip}") {
+      self.get_json("/1.1/address/#{ip}")
     }
-    return res
   end
 
-  private
-  def build_indices(filename)
-    file = File.read(filename)
-    robots = JSON.parse(file)
-    robots.each {|robot|
-      if !robot['cidrs'].nil?
-        robot['cidrs'] = robot['cidrs'].collect {|cidr| cidr2range(cidr)}
-      end
+  def fetch_user_agent(user_agent)
+    self.with_cache("ua-#{Digest::MD5.hexdigest(user_agent)}") {
+      self.post_json("/1.1/user-agent", {:value => user_agent})
     }
-    @ip2robots = group_by_multi(robots, 'ips')
-    @cidr2robots = group_by_multi(robots, 'cidrs')
-    @ip2cidrs = IntervalTree::Tree.new(@cidr2robots.keys)
-    @ua2robots = group_by_multi(robots, 'uas')
   end
 
-  public
-  def register
-    build_indices(@db_path)
+  def fetch_identity(ip, user_agent)
+    ip = ip || ""
+    user_agent = user_agent || ""
+    self.with_cache("identity-#{Digest::MD5.hexdigest(ip)}-#{Digest::MD5.hexdigest(user_agent)}") {
+      self.post_json("/1.1/identity", {:address => ip, :user_agent => user_agent})
+    }
   end
 
-  # Take a User-Agent string and an IP address and return a robot description, or nil.
-  private
-  def detect(ua, ip)
-    # Look for robots with the same IP addressor CIDR
-    ip_candidates = []
-    cidr_candidates = []
-    if ip
-      i = ip.ipv4? ? ip.ipv4_mapped.to_i : ip.to_i # convert IP to arbitrary length integer
-      ip_candidates = @ip2robots[i]
-      cidrs = @ip2cidrs.search(i)
-      cidr_candidates = cidrs.collect {|cidr| @cidr2robots[cidr]}.reduce([], :concat) unless cidrs.nil?
-    end
-    # Look for robots with the same User-Agent
-    ua_candidates = []
-    if ua
-      ua_candidates = @ua2robots[Digest::MD5.hexdigest(ua)]
-    end
-    # Make a final decision
-    robots = ((ip_candidates | cidr_candidates) & ua_candidates)
-    if !robots.empty?
-      robot = robots[0]
-      url = "https://access.watch/database/robots/#{robot['reputation']}/#{robot['urlid'] or robot['id']}"
-      {'identity'   => {'type' => 'robot'},
-       'robot'      => {'id'   => robot['id'],
-                        'name' => robot['name'],
-                        'url'  => url},
-       'reputation' => {'status' => robot['reputation']}}
+  def augment(event, destination, data, keys=nil)
+    if destination && data
+      event.set(destination,
+                data.select {|k, v|
+                  (keys.nil? or keys.include?(k)) && !(v.nil? || v.empty?)
+                })
     end
   end
 
   public
   def filter(event)
-    ip_s = event.get(@ip_source)
-    ip = IPAddr.new ip_s unless ip_s.nil?
-    robot = detect(event.get(@ua_source), ip)
-    if robot
-      event.set('identity',   robot['identity'])   unless robot['identity'].nil?
-      event.set('robot',      robot['robot'])      unless robot['robot'].nil?
-      event.set('reputation', robot['reputation']) unless robot['reputation'].nil?
+    ip = event.get(@ip_source)
+    user_agent = event.get(@user_agent_source)
+    if @ip_source and @user_agent_source
+      response = self.fetch_identity(ip, user_agent)
+      if response[:status] == :success
+        data = response[:data]
+        self.augment(event, @address_destination, data["address"], @@address_keys)
+        self.augment(event, @robot_destination, data["robot"], @@robot_keys)
+        self.augment(event, @reputation_destination, data["reputation"])
+      end
+    elsif @ip_source
+      response = self.fetch_address(ip)
+      if response[:status] == :success
+        self.augment(event, @address_destination, response[:data], @@address_keys)
+      end
+    else
+      response = self.fetch_user_agent(user_agent)
+      if response[:status] == :success
+        self.augment(event, @user_agent_destination, response[:data])
+      end
     end
     filter_matched(event)
   end

data/logstash-filter-accesswatch.gemspec

@@ -1,10 +1,10 @@
 # coding: utf-8
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-accesswatch'
-  s.version       = '0.1.0'
+  s.version       = '0.2.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'The Logstash filter plugin for Access Watch (http://access.watch).'
-  s.description   = 'The Access Watch filter adds information about robots visiting your website based on data from our robots database.'
+  s.description   = 'The Access Watch filter adds information about robots visiting your website based on data from our robot database.'
   s.homepage      = 'http://access.watch'
   s.authors       = ['Benoît Fleury']
   s.email         = 'benoit@access.watch'
@@ -12,16 +12,15 @@ Gem::Specification.new do |s|
 
   # Files
   s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','Gemfile','LICENSE']
-   # Tests
-  s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true',
                  'logstash_group' => 'filter' }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core-plugin-api', '~>   2.0'
-  s.add_runtime_dependency 'augmented_interval_tree',  '~> 0.1.1'
+  s.add_runtime_dependency 'logstash-core-plugin-api',   '~> 2.0'
+  s.add_runtime_dependency 'logstash-mixin-http_client', '~> 5.2'
+  s.add_runtime_dependency 'lru_redux',                  '~> 1.1'
 
   s.add_development_dependency 'logstash-devutils', '1.3.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-accesswatch
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Benoît Fleury
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-18 00:00:00.000000000 Z
+date: 2017-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -29,15 +29,29 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
-  name: augmented_interval_tree
+        version: '5.2'
+  name: logstash-mixin-http_client
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: '5.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  name: lru_redux
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -53,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.3.3
 description: The Access Watch filter adds information about robots visiting your website
-  based on data from our robots database.
+  based on data from our robot database.
 email: benoit@access.watch
 executables: []
 extensions: []