enricher 0.0.2 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ba7623f738d308075b0b07cf6365f83f3f11053b
-  data.tar.gz: 040c1a1349e741d87da53fa7c435841ef8995c34
+  metadata.gz: 7f87c647a272e4dd0b39b45b5e3691fdf3f8fa0e
+  data.tar.gz: 6fc0c075c318729355ee93f56c9982714d39bfad
 SHA512:
-  metadata.gz: 36748a53b45b8ba519090621481ebbeb078d098f42a1fc39dd1bc45483cbab4bfe7a8be3e7360ed531717e51f750f6cf49d347e8b61d7f49b94e52e4b876d33b
-  data.tar.gz: 81e992ba9f680437405ee6369a3c6ae171660684454351c4464133487306c4f678e855efdbaeaffe937bc39ff13a114b98e8a0a98279b40ed571ec51351ceed4
+  metadata.gz: d883c35af6c5d1542aaade6b622f82b5c3044a78633fba1abc250a00c4851c53b2e6aef4c895b84fca7044657c1e7c16dbac34843d17b11bea9ebdb9c685510a
+  data.tar.gz: d88bca4c18a7c68e74490492a8763d5e5b83f300c257d6cf421e289152ed98d161a28e243bff86d2e2084cbca3c63f1df4afbdb4443f150b14e3624a886d6f9b

data/enricher.gemspec

@@ -18,5 +18,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'bundler', '~> 1.0'
   gem.add_dependency "geoip", '~> 1.2'
   gem.add_dependency "netaddr", '~> 1.5'
+  gem.add_dependency "rest-client"
+  gem.add_dependency "json"
 
 end

data/lib/enricher.rb

@@ -4,20 +4,28 @@ require 'ipaddr'
 require 'logger'
 require 'rubygems'
 require 'tempfile'
+require 'net/http'
+require 'date'
 
 # RubyGems
 
 require 'json'
 require 'geoip'
 require 'netaddr'
+require 'rest-client'
+
 
 # Internal 
 module Enricher
   $:.unshift(File.dirname(__FILE__))
+
+  require 'vash'
   require 'enricher/version'
+  require 'enricher/exceptions'
   require 'enricher/bogon'
+  require 'enricher/bgpranking'
   require 'enricher/encoder'
-
+  
   DEBUG=false
   LOGGING=false
 

data/lib/enricher/bgpranking.rb

@@ -0,0 +1,41 @@
+module Enricher
+  #
+  # BGP dynamic ASN ranking checks provided by circl.lu.
+  #
+  # Results cached locally for 12hr. 
+  #
+  # >  r = RestClient.post "http://bgpranking.circl.lu/json", { 'method' => 'cached_daily_rank', 'asn' => 198540, 'date' => '2014-02-23' }.to_json, :content_type => :json, :accept => :json
+  # => "[198540, "ELAN-AS Przedsiebiorstwo Uslug Specjalistycznych ELAN mgr inz. Andrzej Niechcial", "2014-02-23", "global", 1.0496093750000002]"
+  # >>  r = RestClient.post "http://bgpranking.circl.lu/json", { 'method' => 'cached_daily_rank', 'asn' => 198540, 'date' => '2014-02-24' }.to_json, :content_type => :json, :accept => :json
+  # => "[198540, "ELAN-AS Przedsiebiorstwo Uslug Specjalistycznych ELAN mgr inz. Andrzej Niechcial", "2014-02-24", "global", 1.09609375]"
+  # >>  a = JSON.parse(r)
+ 
+  class BGPRanking
+
+    BGP_RANK_URL = "http://bgpranking.circl.lu/json"
+
+	def self.rank?(addr)
+	  asn = addr.strip[/[0-9]+/]	
+	  if asn =~ /[0-9]+/
+	    @@cache ||= Vash.new 
+	    # Voliate Cache store for 43200 (12hr)
+	    @@cache["asn#{asn}".to_sym] ||= self.onlinerank?(asn)
+	  else
+	  	return "0.0"
+      end
+	end
+
+    def self.cache
+    	@@cache
+    end
+
+    private
+
+	def self.onlinerank?(addr)
+	  resp = RestClient.post BGP_RANK_URL, { 'method' => 'cached_daily_rank', 'asn' => addr, 'date' => Date.strptime((Date.today - 1).to_s, '%Y-%m-%d').to_s }.to_json, :content_type => :json, :accept => :json
+	  return "%.6f" % JSON.parse(resp)[4] 
+	end
+
+  end
+
+end

data/lib/enricher/bogon.rb

@@ -1,4 +1,15 @@
 module Enricher
+  #
+  # Bogons ipv4 allow for both static simple checks and for dynamic full Bogon list checks provided by Team Cymru.
+  #
+  # >> @@mybogon = Enricher::Bogon.new(:live)^C
+  # >> @@mybogon.contains?('205.166.22.1')
+  # => true
+  # >> @@mybogon = Enricher::Bogon.new(:ipv4)
+  # => #<Enricher::Bogon:0x00000002fb0368 @bogon=[0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24, 192.168.0.0/16, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24, 224.0.0.0/4, 240.0.0.0/4]>
+  # >> @@mybogon.contains?('205.166.22.1')
+  # => false
+  #
   class Bogon
     
     BOGONIPV4 = ['0.0.0.0/8',
@@ -16,22 +27,36 @@ module Enricher
     '224.0.0.0/4',
     '240.0.0.0/4']
     
+    LIST_URL = "http://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt"
+
     def initialize(bogon)
-      if bogon == :bogonipv4
+      if bogon == :ipv4
         @bogon = BOGONIPV4.collect do |cidr|
           NetAddr::CIDR.create(cidr)
         end
+      elsif bogon == :live
+        @bogon = []
+        Net::HTTP.get(URI.parse(LIST_URL)).each_line do |line|
+          if line !~ /^#/
+            @bogon << NetAddr::CIDR.create(line.strip)
+          end
+        end
       else
-        raise BogonSetUndefined, "Only the :bogonipv4 aggregated set is defined at this time"
+        raise BogonSetUndefined, "Only the :ipv4 aggregated set, and :live via http is defined at this time. illegal use of #{bogon}"
       end
+
+      @bogon
     end  
    
     def contains?(ip)
-      @bogon.each do |net|
-        return true if net.contains?(ip)
-      end
+      @bogon.each { |net| return true if net.contains?(ip) }
       return false
     end
-    
+   
+    def addresses
+      @bogon ||= self.initialize
+    end
+
   end
+
 end

data/lib/enricher/encoder.rb

@@ -1,6 +1,20 @@
 module Enricher 
   
   class Encoder
+  
+    def self.encode(ip)
+      
+      @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
+      @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+      @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoLiteCity.dat")
+      
+      @@bogon_type ||= self.bogon_type
+      @@bogon ||= Bogon.new(@@bogon_type)
+      
+      asn = @@geoASN.asn(ip).number rescue "--"
+      
+      {:ip => IPAddr.new(ip).to_i, :asn => asn, :asn_rank => Enricher::BGPRanking.rank?(asn), :geoip => @@geoCoder.country(ip).country_code3, :bogon => @@bogon.contains?(ip)}
+    end
     
     def self.aton(a)
       IPAddr.new(a).to_i
@@ -9,21 +23,21 @@ module Enricher
     def self.ntoa(a)
       IPAddr.new(a, Socket::AF_INET).to_s
     end
-  
-    def self.encode(ip)
-      @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
-      @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
-      @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoLiteCity.dat")
-      @@bogon ||= Bogon.new(:bogonipv4)
-      asn = @@geoASN.asn(ip).number rescue "--"
-      {:ip => IPAddr.new(ip).to_i, :asn => asn, :geoip => @@geoCoder.country(ip).country_code3, :bogon => @@bogon.contains?(ip)}
+
+    def self.rank?(asn)
+      Enricher::BGPRanking.rank?(asn)
     end
-    
+
     def self.bogon?(ip)
-      @@bogon ||= Bogon.new(:bogonipv4)
+      @@bogon_type ||= self.bogon_type
+      @@bogon ||= Bogon.new(@@bogon_type)
       return @@bogon.contains?(ip)
     end
     
+    def self.bogon_type(bogon_sym=:ipv4)
+      bogon_sym
+    end
+
     def self.asn(ip)
       @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
       return @@geoASN.asn(ip).number rescue "--"

data/lib/enricher/exceptions.rb

@@ -0,0 +1,5 @@
+module Enricher
+ 
+  class BogonSetUndefined < StandardError; end
+
+end  

data/lib/enricher/version.rb

@@ -1,3 +1,3 @@
 module Enricher
-    VERSION = '0.0.2'
+    VERSION = '0.0.4'
 end

data/lib/vash.rb

@@ -0,0 +1,110 @@
+#############################################################################
+# Class: Vash (Ruby Volatile Hash)
+# Hash that returns values only for a short time.  This is useful as a cache
+# where I/O is involved.  The primary goal of this object is to reduce I/O 
+# access and due to the nature of I/O being slower then memory, you should also
+# see a gain in quicker response times.
+#
+# For example, if Person.first found the first person from the database & cache 
+# was an instance of Vash then the following would only contact the database for
+# the first iteration:
+#
+# > cache = Vash.new
+# > 1000.times {cache[:person] ||= Person.first}
+# 
+# However if you did the following immediately following that command it would 
+# hit the database again:
+#
+# > sleep 43201
+# > cache[:person] ||= Person.first
+#
+# The reason is that there is a default Time-To-Live of 43200 seconds.  You can 
+# also set a custom TTL of 10 seconds like so:
+# 
+# > cache[:person, 10] = Person.first
+# 
+# The Vash object will forget any answer that is requested after the specified
+# TTL.  It is a good idea to manually clean things up from time to time because 
+# it is possible that you'll cache data but never again access it and therefor 
+# it will stay in memory after the TTL has expired.  To clean up the Vash object, 
+# call the method: cleanup!
+# 
+# > sleep 11        # At this point the prior person ttl will be expired
+#                   #  but the person key and value will still exist.
+# > cache           # This will still show the the entire set of keys 
+#                   #  regardless of the TTL, the :person will still exist
+# > cache.cleanup!  # All of the TTL's will be inspected and the expired 
+#                   #  :person key will be deleted.
+#
+# The cleanup must be manually called because the purpose of the Vash is to 
+# lessen needless I/O calls and gain speed not to slow it down with regular
+# maintenance.
+class Vash < Hash
+  def initialize(constructor = {})
+    @register ||= {} # remembers expiration time of every key
+    if constructor.is_a?(Hash)
+      super()
+      merge(constructor)
+    else
+      super(constructor)
+    end
+  end
+  
+  alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
+  alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
+  
+  def [](key)
+    sterilize(key)
+    clear(key) if expired?(key)
+    regular_reader(key)
+  end
+  
+  def []=(key, *args)
+    # a little bit o variable hacking to support (h[key, ttl] = value), which will come 
+    # accross as (key, [ttl, value]) whereas (h[key]=value) comes accross as (key, [value])
+    if args.length == 2
+      value, ttl = args[1], args[0]
+    elsif args.length == 1
+      value, ttl = args[0], 43200
+    else
+      raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
+                           "Example Usage:  volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
+    end
+    sterilize(key)
+    ttl(key, ttl)
+    regular_writer(key, value)
+  end
+  
+  def merge(hsh)
+    hsh.map {|key,value| self[sterile(key)] = hsh[key]}
+    self
+  end
+  
+  def cleanup!
+    now = Time.now.to_i
+    @register.map {|k,v| clear(k) if v < now}
+  end
+  
+  def clear(key)
+    sterilize(key)
+    @register.delete key
+    self.delete key
+  end
+  
+private
+  def expired?(key)
+    Time.now.to_i > @register[key].to_i
+  end
+  
+  def ttl(key, secs=43200)
+    @register[key] = Time.now.to_i + secs.to_i
+  end
+  
+  def sterile(key)
+    String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
+  end
+  
+  def sterilize(key)
+    key = sterile(key)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: enricher
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
 platform: ruby
 authors:
 - shadowbq
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-24 00:00:00.000000000 Z
+date: 2014-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Enricher, the IP and URL data enhancer
 email:
 - shadowbq@gmail.com
@@ -66,9 +94,12 @@ files:
 - Rakefile
 - enricher.gemspec
 - lib/enricher.rb
+- lib/enricher/bgpranking.rb
 - lib/enricher/bogon.rb
 - lib/enricher/encoder.rb
+- lib/enricher/exceptions.rb
 - lib/enricher/version.rb
+- lib/vash.rb
 - log/enricher.log
 homepage: https://github.com/shadowbq/enricher
 licenses: