enricher 0.0.4 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f87c647a272e4dd0b39b45b5e3691fdf3f8fa0e
-  data.tar.gz: 6fc0c075c318729355ee93f56c9982714d39bfad
+  metadata.gz: f3a26447430c0aabed962d287dd55ecb300c4b55
+  data.tar.gz: 8afd50139c78292f1cdd4931c966a8f0414675cf
 SHA512:
-  metadata.gz: d883c35af6c5d1542aaade6b622f82b5c3044a78633fba1abc250a00c4851c53b2e6aef4c895b84fca7044657c1e7c16dbac34843d17b11bea9ebdb9c685510a
-  data.tar.gz: d88bca4c18a7c68e74490492a8763d5e5b83f300c257d6cf421e289152ed98d161a28e243bff86d2e2084cbca3c63f1df4afbdb4443f150b14e3624a886d6f9b
+  metadata.gz: 4d7c1bca11cc0e5cec806a22fa4fb878492673c24c479b1b3d0a786c013b09258dec97b355ed4597bca3a1820029a95f388c0b41a01a45669168be13545e8bfb
+  data.tar.gz: 378842b2949ad62ea3f0718600bfccbe0581a3339408e9287e388afb7ebd412d76601c33f72b9ea40677bec02c143df9e70629b20a8910bb50b2720badd85de4

data/README.md

@@ -2,33 +2,142 @@
 
 IPv4 Data Enricher
 
+[![Gem Version](https://badge.fury.io/rb/enricher.png)](http://badge.fury.io/rb/enricher)
+
 ### Static Calculators:
 
-Calculate ASN, CC3, Bogon, and Lat Long. 
+Calculate Maxmind ASN, CC3, Bogon inclusion, and geodata such as Lat Long. 
+
+CDN Hostname check regex lookup.
 
 ### Online Calculators:
 
-IPv4 reputation with VOIDIP
-IPv4 reputation with VirusTotal
+BGP Ranking with http://bgpranking.circl.lu/ ASN Calculator
+
+Full Bogon Checking with Team Cymru List
+
+Reverse DNS Lookups from (L3 DNS Servers 4.2.2.2/3/4)
+
+
+### Usage
+
+```
+>> require 'enricher'
+=> true
+```
+
+#### For Pure Offline Meta Data Enhancement
+```
+>> a = Enricher::Encoder.encode('10.48.185.173')
+=> {:ip=>170965421, :asn=>"--", :asn_rank=>"0.0", :geoip=>"--", :bogon=>true}
+
+>> a = Enricher::Encoder.encode('108.48.185.173')
+=> {:ip=>1815132589, :asn=>"AS701", :asn_rank=>"0.000011", :geoip=>"USA", :bogon=>false}
+```
+
+#### For Online Meta Data Enrichment
+```
+2.1.2 :006 > a = Enricher::Encoder.encode_online('96.3.8.26')
+ => {:ip=>1610811418, :asn=>"AS11232", :asn_rank=>"0.000048", :geoip=>"USA", :bogon=>false, :reverse=>"host-26-8-3-96.midco.net.", :cdn=>false} 
+2.1.2 :007 > a = Enricher::Encoder.encode_online('96.6.113.42')
+ => {:ip=>1611034922, :asn=>"AS20940", :asn_rank=>"0.000411", :geoip=>"USA", :bogon=>false, :reverse=>"a96-6-113-42.deploy.akamaitechnologies.com.", :cdn=>true} 
+2.1.2 :008 > a = Enricher::Encoder.encode_online('119.27.76.185')
+ => {:ip=>1998277817, :asn=>"AS4837", :asn_rank=>"0.000082", :geoip=>"CHN", :bogon=>false, :reverse=>"", :cdn=>false} 
+2.1.2 :009 > a = Enricher::Encoder.encode_online('199.27.76.185')
+ => {:ip=>3340455097, :asn=>"AS54113", :asn_rank=>"0.000526", :geoip=>"USA", :bogon=>false, :reverse=>"", :cdn=>false} 
+2.1.2 :010 > a = Enricher::Encoder.encode_online('54.239.195.35')
+ => {:ip=>921682723, :asn=>"AS16509", :asn_rank=>"0.000293", :geoip=>"USA", :bogon=>false, :reverse=>"server-54-239-195-35.nrt12.r.cloudfront.net.", :cdn=>true} 
+2.1.2 :011 > a = Enricher::Encoder.encode_online('54.239.195.149')
+ => {:ip=>921682837, :asn=>"AS16509", :asn_rank=>"0.000293", :geoip=>"USA", :bogon=>false, :reverse=>"server-54-239-195-149.nrt12.r.cloudfront.net.", :cdn=>true} 
+2.1.2 :012 > a = Enricher::Encoder.encode_online('72.21.91.8')
+ => {:ip=>1209359112, :asn=>"AS15133", :asn_rank=>"0.000312", :geoip=>"USA", :bogon=>false, :reverse=>"", :cdn=>false} 
+2.1.2 :013 > a = Enricher::Encoder.encode_online('173.194.121.44')
+ => {:ip=>2915203372, :asn=>"AS15169", :asn_rank=>"0.000204", :geoip=>"USA", :bogon=>false, :reverse=>"iad23s26-in-f12.1e100.net.", :cdn=>true} 
+2.1.2 :014 > a = Enricher::Encoder.encode_online('205.234.175.175')
+ => {:ip=>3454709679, :asn=>"AS30081", :asn_rank=>"0.000781", :geoip=>"USA", :bogon=>false, :reverse=>"vip1.G-anycast1.cachefly.net.", :cdn=>true} 
+```
+
+### Use of Volatile Hash DB
+
+BGP Scores are stored for 12 hours in a volatile hash cache.
+
+```
+2.1.2 :015 > Enricher::BGPRanking.cache
+ => {:asn12542=>"0.000010", :asn11232=>"0.000048", :asn20940=>"0.000411", :asn4837=>"0.000082", :asn54113=>"0.000526", :asn16509=>"0.000293", :asn15133=>"0.000312", :asn15169=>"0.000204", :asn30081=>"0.000781"} 
+```
+
+### TODO
 
-BGP Ranking with Cyrmu ASN Calculator
+* IPv4 reputation with VOIDIP
+* IPv4 reputation with VirusTotal (uirusu)
 
 ## Requirements
 
-Intenet connectivity for Online Calculators.
+Intenet connectivity is required *only* for Online Meta Data Enhancement.
+
+### BootStrapping GeoIP
+
+There is a rake command include to bootstrap the fetching of geoip data if you are a sudo enabled linux user.
+
+```
+(/home/shadowbq/.rvm/gems/ruby-2.1.2/gems/enricher-0.0.7)$ rake fetch_geoip_data
+--2014-12-31 11:32:28--  http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
+Resolving geolite.maxmind.com (geolite.maxmind.com)... 141.101.115.190, 141.101.114.190, 2400:cb00:2048:1::8d65:72be, ...
+Connecting to geolite.maxmind.com (geolite.maxmind.com)|141.101.115.190|:80... connected.
+HTTP request sent, awaiting response... 200 OK
+Length: 428181 (418K) [application/octet-stream]
+Saving to: ‘/usr/local/lib/share/enricher/GeoIP.dat.gz’
+
+100%[=====================================================================================================================================================================================================================================>] 428,181     --.-K/s   in 0.08s   
+
+2014-12-31 11:32:28 (5.22 MB/s) - ‘/usr/local/lib/share/enricher/GeoIP.dat.gz’ saved [428181/428181]
+
+--2014-12-31 11:32:28--  http://geolite.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
+Resolving geolite.maxmind.com (geolite.maxmind.com)... 141.101.114.190, 141.101.115.190, 2400:cb00:2048:1::8d65:73be, ...
+Connecting to geolite.maxmind.com (geolite.maxmind.com)|141.101.114.190|:80... connected.
+HTTP request sent, awaiting response... 200 OK
+Length: 2056964 (2.0M) [application/octet-stream]
+Saving to: ‘/usr/local/lib/share/enricher/GeoIPASNum.dat.gz’
+
+100%[=====================================================================================================================================================================================================================================>] 2,056,964   1.38MB/s   in 1.4s   
+
+2014-12-31 11:32:30 (1.38 MB/s) - ‘/usr/local/lib/share/enricher/GeoIPASNum.dat.gz’ saved [2056964/2056964]
+
+--2014-12-31 11:32:30--  http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
+Resolving geolite.maxmind.com (geolite.maxmind.com)... 141.101.115.190, 141.101.114.190, 2400:cb00:2048:1::8d65:72be, ...
+Connecting to geolite.maxmind.com (geolite.maxmind.com)|141.101.115.190|:80... connected.
+HTTP request sent, awaiting response... 200 OK
+Length: 11896864 (11M) [application/octet-stream]
+Saving to: ‘/usr/local/lib/share/enricher/GeoLiteCity.dat.gz’
+
+100%[=====================================================================================================================================================================================================================================>] 11,896,864  5.10MB/s   in 2.2s   
+
+2014-12-31 11:32:32 (5.10 MB/s) - ‘/usr/local/lib/share/enricher/GeoLiteCity.dat.gz’ saved [11896864/11896864]
+
+>$ ls -la /usr/local/lib/share/enricher/
+total 22664
+drwxr-xr-x 2 root root     4096 Dec 31 11:32 .
+drwxr-xr-x 3 root root     4096 Dec 31 11:23 ..
+-rw-r--r-- 1 root root   748606 Dec  2 16:43 GeoIP.dat
+-rw-r--r-- 1 root root  3766172 Dec 12 17:54 GeoIPASNum.dat
+-rw-r--r-- 1 root root 18678957 Dec  2 16:57 GeoLiteCity.dat
+
+```
+
+(Manual Methods)
 
-Maxmind dat file location requirement: `/usr/local/lib/share/enricher`
+Maxmind(R) dat file location requirement: `/usr/local/lib/share/enricher`
 
 You need to download and install each of the free GeoLite country, city or ASN databases, or a subscription database version. 
 
 The last known download locations for the GeoLite database versions are:
 
-<geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz>
-<geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz>
-<geolite.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz>
+* http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz
+* http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
+* http://geolite.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz
 
 
-## Automating the update of Static Content (via crontrab)
+### Automating the update of Static Content (via crontrab)
 
 We can add a cron-job to automate the monthly process of updating the GeoIP database:
 

data/Rakefile

@@ -8,4 +8,12 @@ Rake::TestTask.new do |test|
   test.libs << "test" 
   test.test_files = Dir[ "test/test_*.rb" ]
   test.verbose = true
-end
+end
+
+desc 'Fetch the GeoIP Data sets'
+task :fetch_geoip_data do 
+  `sudo mkdir -p /usr/local/lib/share/enricher`
+  `sudo sh -c '/usr/bin/wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz -O /usr/local/lib/share/enricher/GeoIP.dat.gz; /bin/gunzip -f /usr/local/lib/share/enricher/GeoIP.dat.gz'`
+  `sudo sh -c '/usr/bin/wget http://geolite.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz -O /usr/local/lib/share/enricher/GeoIPASNum.dat.gz; /bin/gunzip -f /usr/local/lib/share/enricher/GeoIPASNum.dat.gz'`
+  `sudo sh -c '/usr/bin/wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz -O /usr/local/lib/share/enricher/GeoLiteCity.dat.gz; /bin/gunzip -f /usr/local/lib/share/enricher/GeoLiteCity.dat.gz'`
+end

data/enricher.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'bundler', '~> 1.0'
   gem.add_dependency "geoip", '~> 1.2'
   gem.add_dependency "netaddr", '~> 1.5'
+  gem.add_dependency "net-dns", '~> 0.8'
   gem.add_dependency "rest-client"
   gem.add_dependency "json"
 

data/lib/enricher.rb

@@ -13,7 +13,8 @@ require 'json'
 require 'geoip'
 require 'netaddr'
 require 'rest-client'
-
+require 'net/dns'
+#require 'uirusu'
 
 # Internal 
 module Enricher
@@ -24,14 +25,24 @@ module Enricher
   require 'enricher/exceptions'
   require 'enricher/bogon'
   require 'enricher/bgpranking'
+  require 'enricher/cdn'
+  require 'enricher/resolver'
   require 'enricher/encoder'
   
   DEBUG=false
   LOGGING=false
-
+  COMMON_DATA_PATHS=[
+    '/usr/local/lib/share/enricher', 
+    '/usr/local/share/enricher',
+    '/usr/local/lib/enricher',
+    '/usr/local/etc/enricher',
+    '/etc/enricher',
+    '/var/db/enricher'
+  ]
+  
   #Setup Paths
   LIB_PATH = File.expand_path("../", __FILE__)
-  CONFIG_PATH = File.expand_path("../../db", __FILE__)
+  #CONFIG_PATH = File.expand_path("../../db", __FILE__)
   
   if Enricher::DEBUG
     Enricher::DATA_PATH = File.expand_path("../../data", __FILE__)
@@ -39,11 +50,18 @@ module Enricher
       Enricher::LOG_PATH = File.expand_path("../../log", __FILE__)
       logfile = "#{Enricher::LOG_PATH}/enricher.log"
     end   
-  else  
-    Enricher::DATA_PATH = File.path("/usr/local/lib/share/enricher")
-    if Enricher::LOGGING  
-      logfile = Tempfile.new('enricher.log')
-      Enricher::LOG_PATH = File.dirname(logfile.path) 
+  else
+    COMMON_DATA_PATHS.each do |dirname| 
+      if File.exists?(dirname) && File.directory?(dirname)
+        Enricher::DATA_PATH = File.path(dirname)
+        if Enricher::LOGGING  
+          logfile = Tempfile.new('enricher.log')
+          Enricher::LOG_PATH = File.dirname(logfile.path) 
+        end
+        break 
+      else
+        raise EnricherPathMissing, "Enricher data path not found in Common Data Paths. (i.e /usr/local/lib/share/enricher) See README.md"
+      end
     end   
   end   
 

data/lib/enricher/bgpranking.rb

@@ -9,33 +9,31 @@ module Enricher
   # >>  r = RestClient.post "http://bgpranking.circl.lu/json", { 'method' => 'cached_daily_rank', 'asn' => 198540, 'date' => '2014-02-24' }.to_json, :content_type => :json, :accept => :json
   # => "[198540, "ELAN-AS Przedsiebiorstwo Uslug Specjalistycznych ELAN mgr inz. Andrzej Niechcial", "2014-02-24", "global", 1.09609375]"
   # >>  a = JSON.parse(r)
- 
   class BGPRanking
 
     BGP_RANK_URL = "http://bgpranking.circl.lu/json"
 
-	def self.rank?(addr)
-	  asn = addr.strip[/[0-9]+/]	
-	  if asn =~ /[0-9]+/
-	    @@cache ||= Vash.new 
-	    # Voliate Cache store for 43200 (12hr)
-	    @@cache["asn#{asn}".to_sym] ||= self.onlinerank?(asn)
-	  else
-	  	return "0.0"
+    def self.rank?(addr)
+      asn = addr.strip[/[0-9]+/]  
+      if asn =~ /[0-9]+/
+        @@cache ||= Vash.new 
+        # Voliate Cache store for 43200 (12hr)
+        @@cache["asn#{asn}".to_sym] ||= self.onlinerank?(asn)
+      else
+        return "0.0"
       end
-	end
+    end
 
     def self.cache
-    	@@cache
+      @@cache
     end
 
     private
 
-	def self.onlinerank?(addr)
-	  resp = RestClient.post BGP_RANK_URL, { 'method' => 'cached_daily_rank', 'asn' => addr, 'date' => Date.strptime((Date.today - 1).to_s, '%Y-%m-%d').to_s }.to_json, :content_type => :json, :accept => :json
-	  return "%.6f" % JSON.parse(resp)[4] 
-	end
+    def self.onlinerank?(addr)
+      resp = RestClient.post BGP_RANK_URL, { 'method' => 'cached_daily_rank', 'asn' => addr, 'date' => Date.strptime((Date.today - 1).to_s, '%Y-%m-%d').to_s }.to_json, :content_type => :json, :accept => :json
+      return "%.6f" % JSON.parse(resp)[4] 
+    end
 
   end
-
-end
+end

data/lib/enricher/cdn.rb

@@ -0,0 +1,112 @@
+module Enricher
+  #
+  # Bogons ipv4 allow for both static simple checks and for dynamic full Bogon list checks provided by Team Cymru.
+  #
+  # CDN List provided by - (23 Dec 2014) https://github.com/WPO-Foundation/webpagetest/blob/master/agent/wpthook/cdn.h
+  #
+  class CDN
+    
+    CDN_PROVIDERS = [
+      [".akamai.net", "Akamai"],
+      [".akamaiedge.net", "Akamai"],
+      [".akamaihd.net", "Akamai"],
+      [".edgesuite.net", "Akamai"],
+      [".edgekey.net", "Akamai"],
+      [".srip.ne", "Akamai"],
+      [".akamaitechnologies.com", "Akamai"],
+      [".akamaitechnologies.fr", "Akamai"],
+      [".llnwd.net", "Limelight"],
+      ["edgecastcdn.net", "Edgecast"],
+      [".systemcdn.net", "Edgecast"],
+      [".transactcdn.net", "Edgecast"],
+      [".v1cdn.net", "Edgecast"],
+      [".v2cdn.net", "Edgecast"],
+      [".v3cdn.net", "Edgecast"],
+      [".v4cdn.net", "Edgecast"],
+      [".v5cdn.net", "Edgecast"],
+      ["hwcdn.net", "Highwinds"],
+      [".simplecdn.net", "Simple CDN"],
+      [".instacontent.net", "Mirror Image"],
+      [".footprint.net", "Level 3"],
+      [".ay1.b.yahoo.com", "Yahoo"],
+      [".yimg.", "Yahoo"],
+      [".yahooapis.com", "Yahoo"],
+      [".google.", "Google"],
+      ["googlesyndication.", "Google"],
+      ["youtube.", "Google"],
+      [".googleusercontent.com", "Google"],
+      ["googlehosted.com", "Google"],
+      [".gstatic.com", "Google"],
+      [".insnw.net", "Instart Logic"],
+      [".inscname.net", "Instart Logic"],
+      [".internapcdn.net", "Internap"],
+      [".cloudfront.net", "Amazon CloudFront"],
+      [".netdna-cdn.com", "NetDNA"],
+      [".netdna-ssl.com", "NetDNA"],
+      [".netdna.com", "NetDNA"],
+      [".cotcdn.net", "Cotendo CDN"],
+      [".cachefly.net", "Cachefly"],
+      ["bo.lt", "BO.LT"],
+      [".cloudflare.com", "Cloudflare"],
+      [".afxcdn.net", "afxcdn.net"],
+      [".lxdns.com", "ChinaNetCenter"],
+      [".att-dsa.net", "AT&T"],
+      [".vo.msecnd.net", "Windows Azure"],
+      [".voxcdn.net", "VoxCDN"],
+      [".bluehatnetwork.com", "Blue Hat Network"],
+      [".swiftcdn1.com", "SwiftCDN"],
+      [".cdngc.net", "CDNetworks"],
+      [".gccdn.net", "CDNetworks"],
+      [".panthercdn.com", "CDNetworks"],
+      [".fastly.net", "Fastly"],
+      [".nocookie.net", "Fastly"],
+      [".gslb.taobao.com", "Taobao"],
+      [".gslb.tbcache.com", "Alimama"],
+      [".mirror-image.net", "Mirror Image"],
+      [".yottaa.net", "Yottaa"],
+      [".cubecdn.net", "cubeCDN"],
+      [".r.cdn77.net", "CDN77"],
+      [".incapdns.net", "Incapsula"],
+      [".bitgravity.com", "BitGravity"],
+      [".r.worldcdn.net", "OnApp"],
+      [".r.worldssl.net", "OnApp"],
+      ["tbcdn.cn", "Taobao"],
+      [".taobaocdn.com", "Taobao"],
+      [".ngenix.net", "NGENIX"],
+      [".pagerain.net", "PageRain"],
+      [".ccgslb.com", "ChinaCache"],
+      ["cdn.sfr.net", "SFR"],
+      [".azioncdn.net", "Azion"],
+      [".azioncdn.com", "Azion"],
+      [".azion.net", "Azion"],
+      [".cdncloud.net.au", "MediaCloud"],
+      [".rncdn1.com", "Reflected Networks"],
+      [".cdnsun.net", "CDNsun"],
+      [".mncdn.com", "Medianova"],
+      [".mncdn.net", "Medianova"],
+      [".mncdn.org", "Medianova"],
+      ["cdn.jsdelivr.net", "jsDelivr"],
+      [".nyiftw.net", "NYI FTW"],
+      [".nyiftw.com", "NYI FTW"],
+      [".resrc.it", "ReSRC.it"],
+      [".zenedge.net", "Zenedge"],
+      [".lswcdn.net", "LeaseWeb CDN"],  
+      [".revcn.net", "Rev Software"],
+      [".revdn.net", "Rev Software"],
+      [".1e100.net", "Google"]
+    ]
+
+    # Expects a hostname string as a variable
+    def self.contains?(hostname = "")
+      if hostname != "" then 
+        CDN_PROVIDERS.each { |cdn_entry| 
+          if hostname.match(cdn_entry[0]) then
+            return true 
+          end  
+        }
+      end  
+      return false
+    end
+
+  end
+end

data/lib/enricher/encoder.rb

@@ -1,70 +1,99 @@
 module Enricher 
-  
+
   class Encoder
-  
-    def self.encode(ip)
-      
-      @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
-      @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
-      @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoLiteCity.dat")
+
+    ## Class Methods for the Encoder.. 
+    class << self
+
+
+      def encode(ip)
+        
+        @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
+        @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+        @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoLiteCity.dat")
+        @@res ||= Enricher::Resolver.new
+        
+        @@bogon_type ||= self.bogon_type
+        @@bogon ||= Bogon.new(@@bogon_type)
+        
+        asn = @@geoASN.asn(ip).number rescue "--"
+        
+        {:ip => IPAddr.new(ip).to_i, :asn => asn, :asn_rank => Enricher::BGPRanking.rank?(asn), :geoip => @@geoCoder.country(ip).country_code3, :bogon => @@bogon.contains?(ip)}
+      end
       
-      @@bogon_type ||= self.bogon_type
-      @@bogon ||= Bogon.new(@@bogon_type)
+      def encode_online(ip)
+        @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
+        @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+        @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoLiteCity.dat")
+        @@res ||= Enricher::Resolver.new
+        
+        @@bogon_type ||= self.bogon_type(:live)
+        @@bogon ||= Bogon.new(@@bogon_type)
+        
+        asn = @@geoASN.asn(ip).number rescue "--"
+        reverse_hostname = self.reverse(ip) rescue ""
+
+        {:ip => IPAddr.new(ip).to_i, :asn => asn, :asn_rank => Enricher::BGPRanking.rank?(asn), :geoip => @@geoCoder.country(ip).country_code3, :bogon => @@bogon.contains?(ip), :reverse => reverse_hostname, :cdn => self.cdn?(reverse_hostname)}
+      end
+
+      def aton(a)
+        IPAddr.new(a).to_i
+      end
+        
+      def ntoa(a)
+        IPAddr.new(a, Socket::AF_INET).to_s
+      end
+
+      def rank?(asn)
+        Enricher::BGPRanking.rank?(asn)
+      end
+
+      def bogon?(ip)
+        @@bogon_type ||= self.bogon_type
+        @@bogon ||= Bogon.new(@@bogon_type)
+        return @@bogon.contains?(ip)
+      end
       
-      asn = @@geoASN.asn(ip).number rescue "--"
+      def bogon_type(bogon_sym=:ipv4)
+        bogon_sym
+      end
+
+      def asn(ip)
+        @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
+        return @@geoASN.asn(ip).number rescue "--"
+      end
       
-      {:ip => IPAddr.new(ip).to_i, :asn => asn, :asn_rank => Enricher::BGPRanking.rank?(asn), :geoip => @@geoCoder.country(ip).country_code3, :bogon => @@bogon.contains?(ip)}
-    end
-    
-    def self.aton(a)
-      IPAddr.new(a).to_i
-    end
+      def asn_company(ip)
+        @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
+        return @@geoASN.asn(ip).asn rescue "--"
+      end
       
-    def self.ntoa(a)
-      IPAddr.new(a, Socket::AF_INET).to_s
-    end
+      def cc3(ip)
+        @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+        return @@geoCoder.country(ip).country_code3
+      end
 
-    def self.rank?(asn)
-      Enricher::BGPRanking.rank?(asn)
-    end
+      def latitude(ip)
+        @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+        return @@geoCoderCity.city(ip).latitude
+      end
 
-    def self.bogon?(ip)
-      @@bogon_type ||= self.bogon_type
-      @@bogon ||= Bogon.new(@@bogon_type)
-      return @@bogon.contains?(ip)
-    end
-    
-    def self.bogon_type(bogon_sym=:ipv4)
-      bogon_sym
-    end
+      def longitude(ip)
+        @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+        return @@geoCoderCity.city(ip).longitude
+      end
 
-    def self.asn(ip)
-      @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
-      return @@geoASN.asn(ip).number rescue "--"
-    end
-    
-    def self.asn_company(ip)
-      @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
-      return @@geoASN.asn(ip).asn rescue "--"
-    end
-    
-    def self.cc3(ip)
-      @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
-      return @@geoCoder.country(ip).country_code3
-    end
+      def reverse(ip)
+        @@res ||= Enricher::Resolver.new
+        return @@res.reverse?(ip)
+      end
 
-    def self.latitude(ip)
-      @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
-      return @@geoCoderCity.city(ip).latitude
-    end
+      def cdn?(hostname)
+        return Enricher::CDN.contains?(hostname) 
+      end
 
-    def self.longitude(ip)
-      @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
-      return @@geoCoderCity.city(ip).longitude
     end
 
-
   end
 
-end
-
+end

data/lib/enricher/exceptions.rb

@@ -1,5 +1,5 @@
 module Enricher
- 
   class BogonSetUndefined < StandardError; end
-
-end  
+  class EnricherPathMissing < StandardError; end
+  class DisabledClassIncluded < StandardError; end
+end

data/lib/enricher/ipvoid.rb

@@ -0,0 +1,78 @@
+module Enricher
+  #
+  # IPVOID ipv4 allow for dynamic checks against the list checks provided by IPVOID.
+  #
+  class IPVoid
+
+    DISABLED = true
+
+    def self.url_cache
+      @@url_cache
+    end
+
+    def self.hash_cache
+      @@hash_cache
+    end
+
+    def initialize(constructor = {})
+      
+      raise DisabledClassIncluded if DISABLED 
+      #First you need to include the correct require files
+      APT_KEY = "YOUR API KEY HERE"
+      @@hash_cache ||= Vash.new
+      @@url_cache ||= Vash.new  
+        # Voliate Cache store for 43200 (12hr)
+    end
+    
+    def junk
+
+      # RestClient scrape with Nokogiri.... (nokogiri requires libxml which is native which is not jruby compliant.. )
+
+=begin 
+      for ip in open(conf.iplist, "r"):
+    url = "http://www.ipvoid.com/scan/%s" % (ip)
+    emailBody = emailBody + "IP: "+ip
+    resp = requests.get(url)
+    string1 = unicodedata.normalize('NFKD', resp.text).encode('ascii','ignore')
+    r = string1.translate(string.maketrans("\n\t\r", "   "))
+    blacklist = re.search(r'Blacklist Status</td><td><span.+>(\w.+)</span>', r)
+    if blacklist != None and blacklist.group(1) == "BLACKLISTED":
+         emailBody = emailBody + 'The IP is blacklisted! \n'
+         detection = re.search(r'Detection Ratio</td><td>(\d+ / \d+) \(<font', r)
+         emailBody = emailBody + 'Detection Ratio was %s \n' % detection.group(1)
+         detected_line = re.search(r'\s+<tr><td><img src="(.+)', r)
+         detected_sites = re.findall(r'Favicon" />(.+?)</td><td><img src=".+?" alt="Alert" title="Detected!".+?"nofollow" href="(.+?)" title', detected_line.group(1))
+         for site in detected_sites:
+             emailBody = emailBody + "List Name:" + site[0] + "Url: "+ site[1] + "\n\n"
+    else:
+         emailBody = emailBody + 'Not blacklisted...\n\n'
+=end
+    end      
+
+
+    def hash(hash)
+      #To query a hash(sha1/sha256/md5)
+      @@hash_cache["vt_#{hash}".to_sym] ||= Uirusu::VTFile.query_report(VT_APT_KEY, hash)
+      result = Uirusu::VTResult.new(hash, results)
+      result.to_json
+    end
+
+    def url(url)
+
+      # Use Base 36 for symbols
+      #>> "joe@momma.org".hash.to_s(36)
+      #=> "37zed965f04p"
+      #>> "http://joe@momma.org".hash.to_s(36)
+      #=> "vj36lppwievl"
+      #=> Tack on.. vt_ to url converted .hash.to_s(36)
+
+      @@url_cache["vt_#{url.hash.to_s(36)}".to_sym] ||= Uirusu::VTUrl.query_report(VT_APT_KEY, url)
+       
+      result = Uirusu::VTResult.new(url, results)
+      result.to_json
+    end
+
+
+  end
+
+end  

data/lib/enricher/resolver.rb

@@ -0,0 +1,30 @@
+module Enricher
+  #
+  # Bogons ipv4 allow for both static simple checks and for dynamic full Bogon list checks provided by Team Cymru.
+  #
+  # >> @@mybogon = Enricher::Bogon.new(:live)^C
+  # >> @@mybogon.contains?('205.166.22.1')
+  # => true
+  # >> @@mybogon = Enricher::Bogon.new(:ipv4)
+  # => #<Enricher::Bogon:0x00000002fb0368 @bogon=[0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24, 192.168.0.0/16, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24, 224.0.0.0/4, 240.0.0.0/4]>
+  # >> @@mybogon.contains?('205.166.22.1')
+  # => false
+  #
+  class Resolver
+    
+    def initialize(nameservers = ["4.2.2.2","4.2.2.3","4.2.2.4"])
+      @res = Net::DNS::Resolver.new
+      @res.nameservers = nameservers
+    end
+
+    def reverse?(ip)
+      begin
+        packet = @res.search(ip)
+        return packet.answer[0].ptr
+      rescue
+        return ""
+      end
+    end
+
+  end
+end

data/lib/enricher/version.rb

@@ -1,3 +1,3 @@
 module Enricher
-    VERSION = '0.0.4'
-end
+    VERSION = '0.0.7'
+end

data/lib/enricher/virustotal.rb

@@ -0,0 +1,51 @@
+module Enricher
+  #
+  # Bogons ipv4 allow for both static simple checks and for dynamic full Bogon list checks provided by Team Cymru.
+  #
+  class VirusTotal
+
+    DISABLED = true
+    
+    def self.url_cache
+      @@url_cache
+    end
+
+    def self.hash_cache
+      @@hash_cache
+    end
+
+    def initialize(constructor = {})
+
+      raise DisabledClassIncluded if DISABLED 
+      #First you need to include the correct require files
+      APT_KEY = "YOUR API KEY HERE"
+      @@hash_cache ||= Vash.new
+      @@url_cache ||= Vash.new  
+      # Voliate Cache store for 43200 (12hr)
+    end
+    
+    def hash(hash)
+      #To query a hash(sha1/sha256/md5)
+      @@hash_cache["vt_#{hash}".to_sym] ||= Uirusu::VTFile.query_report(VT_APT_KEY, hash)
+      result = Uirusu::VTResult.new(hash, results)
+      result.to_json
+    end
+
+    def url(url)
+
+      # Use Base 36 for symbols
+      #>> "joe@momma.org".hash.to_s(36)
+      #=> "37zed965f04p"
+      #>> "http://joe@momma.org".hash.to_s(36)
+      #=> "vj36lppwievl"
+          #=> Tack on.. vt_ to url converted .hash.to_s(36)
+
+      @@url_cache["vt_#{url.hash.to_s(36)}".to_sym] ||= Uirusu::VTUrl.query_report(VT_APT_KEY, url)
+       
+      result = Uirusu::VTResult.new(url, results)
+      result.to_json
+    end
+
+  end
+
+end  

data/lib/vash.rb

@@ -91,7 +91,8 @@ class Vash < Hash
     self.delete key
   end
   
-private
+  private
+  
   def expired?(key)
     Time.now.to_i > @register[key].to_i
   end
@@ -107,4 +108,5 @@ private
   def sterilize(key)
     key = sterile(key)
   end
+  
 end

metadata

@@ -1,83 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: enricher
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.7
 platform: ruby
 authors:
 - shadowbq
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-25 00:00:00.000000000 Z
+date: 2014-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: geoip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
   name: netaddr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: net-dns
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Enricher, the IP and URL data enhancer
@@ -87,7 +101,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE
 - README.md
@@ -96,9 +110,13 @@ files:
 - lib/enricher.rb
 - lib/enricher/bgpranking.rb
 - lib/enricher/bogon.rb
+- lib/enricher/cdn.rb
 - lib/enricher/encoder.rb
 - lib/enricher/exceptions.rb
+- lib/enricher/ipvoid.rb
+- lib/enricher/resolver.rb
 - lib/enricher/version.rb
+- lib/enricher/virustotal.rb
 - lib/vash.rb
 - log/enricher.log
 homepage: https://github.com/shadowbq/enricher
@@ -111,12 +129,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []