enricher 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ba7623f738d308075b0b07cf6365f83f3f11053b
+  data.tar.gz: 040c1a1349e741d87da53fa7c435841ef8995c34
+SHA512:
+  metadata.gz: 36748a53b45b8ba519090621481ebbeb078d098f42a1fc39dd1bc45483cbab4bfe7a8be3e7360ed531717e51f750f6cf49d347e8b61d7f49b94e52e4b876d33b
+  data.tar.gz: 81e992ba9f680437405ee6369a3c6ae171660684454351c4464133487306c4f678e855efdbaeaffe937bc39ff13a114b98e8a0a98279b40ed571ec51351ceed4

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+*.dat
+.bundle
+.config
+build.sh
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+gem "rake"
+
+group :development
+  gem "bump",             "~> 0.3"
+  gem "colored",          "~> 1.2"
+end
+
+group :test do
+  gem 'minitest-colorize'
+  gem 'turn'
+  gem 'ansi'
+  gem 'simplecov', :require => false
+end
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 shadowbq
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,54 @@
+## Readme
+
+IPv4 Data Enricher
+
+### Static Calculators:
+
+Calculate ASN, CC3, Bogon, and Lat Long. 
+
+### Online Calculators:
+
+IPv4 reputation with VOIDIP
+IPv4 reputation with VirusTotal
+
+BGP Ranking with Cyrmu ASN Calculator
+
+## Requirements
+
+Intenet connectivity for Online Calculators.
+
+Maxmind dat file location requirement: `/usr/local/lib/share/enricher`
+
+You need to download and install each of the free GeoLite country, city or ASN databases, or a subscription database version. 
+
+The last known download locations for the GeoLite database versions are:
+
+<geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz>
+<geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz>
+<geolite.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz>
+
+
+## Automating the update of Static Content (via crontrab)
+
+We can add a cron-job to automate the monthly process of updating the GeoIP database:
+
+```
+ mkdir -p /etc/crontab
+```
+
+Now we will add a custom job:
+
+```
+ vim /etc/crontab/91_Update_GeoIP_db
+```
+
+Add the following to this jopb, this will download and extract the new databases every month:
+
+``` 
+ # Updating the GeoIP database monthly on the 5th at 0:00h.
+ 0 0 5 * * root /usr/bin/wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz -O /usr/local/lib/share/enricher/GeoIP.dat.gz; /bin/gunzip -f /usr/local/lib/share/enricher/GeoIP.dat.gz
+
+ 0 0 5 * * root /usr/bin/wget http://geolite.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz -O /usr/local/lib/share/enricher/GeoIPASNum.dat.gz; /bin/gunzip -f /usr/local/lib/share/enricher/GeoIPASNum.dat.gz
+
+ 0 0 5 * * root /usr/bin/wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz -O /usr/local/lib/share/enricher/GeoLiteCity.dat.gz; /bin/gunzip -f /usr/local/lib/share/enricher/GeoLiteCity.dat.gz
+```

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require "rake/testtask" 
+
+task :default => [:test]
+
+Rake::TestTask.new do |test|
+  test.libs << "test" 
+  test.test_files = Dir[ "test/test_*.rb" ]
+  test.verbose = true
+end

data/enricher.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/enricher/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["shadowbq"]
+  gem.email         = ["shadowbq@gmail.com"]
+  gem.description   = %q{Enricher, the IP and URL data enhancer}
+  gem.summary       = gem.description
+  gem.homepage      = "https://github.com/shadowbq/enricher"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "enricher"
+  gem.require_paths = ["lib"]
+  gem.version       = Enricher::VERSION
+  gem.licenses      = ["MIT"]
+  gem.add_development_dependency 'bundler', '~> 1.0'
+  gem.add_dependency "geoip", '~> 1.2'
+  gem.add_dependency "netaddr", '~> 1.5'
+
+end

data/lib/enricher.rb

@@ -0,0 +1,44 @@
+#STDLIBS
+require 'uri'
+require 'ipaddr'
+require 'logger'
+require 'rubygems'
+require 'tempfile'
+
+# RubyGems
+
+require 'json'
+require 'geoip'
+require 'netaddr'
+
+# Internal 
+module Enricher
+  $:.unshift(File.dirname(__FILE__))
+  require 'enricher/version'
+  require 'enricher/bogon'
+  require 'enricher/encoder'
+
+  DEBUG=false
+  LOGGING=false
+
+  #Setup Paths
+  LIB_PATH = File.expand_path("../", __FILE__)
+  CONFIG_PATH = File.expand_path("../../db", __FILE__)
+  
+  if Enricher::DEBUG
+    Enricher::DATA_PATH = File.expand_path("../../data", __FILE__)
+    if Enricher::LOGGING
+      Enricher::LOG_PATH = File.expand_path("../../log", __FILE__)
+      logfile = "#{Enricher::LOG_PATH}/enricher.log"
+    end   
+  else  
+    Enricher::DATA_PATH = File.path("/usr/local/lib/share/enricher")
+    if Enricher::LOGGING  
+      logfile = Tempfile.new('enricher.log')
+      Enricher::LOG_PATH = File.dirname(logfile.path) 
+    end   
+  end   
+
+  Logger = Logger.new(logfile)
+  
+end

data/lib/enricher/bogon.rb

@@ -0,0 +1,37 @@
+module Enricher
+  class Bogon
+    
+    BOGONIPV4 = ['0.0.0.0/8',
+    '10.0.0.0/8',
+    '100.64.0.0/10',
+    '127.0.0.0/8',
+    '169.254.0.0/16',
+    '172.16.0.0/12',
+    '192.0.0.0/24',
+    '192.0.2.0/24',
+    '192.168.0.0/16',
+    '198.18.0.0/15',
+    '198.51.100.0/24',
+    '203.0.113.0/24',
+    '224.0.0.0/4',
+    '240.0.0.0/4']
+    
+    def initialize(bogon)
+      if bogon == :bogonipv4
+        @bogon = BOGONIPV4.collect do |cidr|
+          NetAddr::CIDR.create(cidr)
+        end
+      else
+        raise BogonSetUndefined, "Only the :bogonipv4 aggregated set is defined at this time"
+      end
+    end  
+   
+    def contains?(ip)
+      @bogon.each do |net|
+        return true if net.contains?(ip)
+      end
+      return false
+    end
+    
+  end
+end

data/lib/enricher/encoder.rb

@@ -0,0 +1,56 @@
+module Enricher 
+  
+  class Encoder
+    
+    def self.aton(a)
+      IPAddr.new(a).to_i
+    end
+      
+    def self.ntoa(a)
+      IPAddr.new(a, Socket::AF_INET).to_s
+    end
+  
+    def self.encode(ip)
+      @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
+      @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+      @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoLiteCity.dat")
+      @@bogon ||= Bogon.new(:bogonipv4)
+      asn = @@geoASN.asn(ip).number rescue "--"
+      {:ip => IPAddr.new(ip).to_i, :asn => asn, :geoip => @@geoCoder.country(ip).country_code3, :bogon => @@bogon.contains?(ip)}
+    end
+    
+    def self.bogon?(ip)
+      @@bogon ||= Bogon.new(:bogonipv4)
+      return @@bogon.contains?(ip)
+    end
+    
+    def self.asn(ip)
+      @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
+      return @@geoASN.asn(ip).number rescue "--"
+    end
+    
+    def self.asn_company(ip)
+      @@geoASN ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIPASNum.dat")
+      return @@geoASN.asn(ip).asn rescue "--"
+    end
+    
+    def self.cc3(ip)
+      @@geoCoder ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+      return @@geoCoder.country(ip).country_code3
+    end
+
+    def self.latitude(ip)
+      @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+      return @@geoCoderCity.city(ip).latitude
+    end
+
+    def self.longitude(ip)
+      @@geoCoderCity ||= GeoIP.new("#{Enricher::DATA_PATH}/GeoIP.dat")
+      return @@geoCoderCity.city(ip).longitude
+    end
+
+
+  end
+
+end
+

data/lib/enricher/version.rb

@@ -0,0 +1,3 @@
+module Enricher
+    VERSION = '0.0.2'
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: enricher
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- shadowbq
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: geoip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: netaddr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+description: Enricher, the IP and URL data enhancer
+email:
+- shadowbq@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- enricher.gemspec
+- lib/enricher.rb
+- lib/enricher/bogon.rb
+- lib/enricher/encoder.rb
+- lib/enricher/version.rb
+- log/enricher.log
+homepage: https://github.com/shadowbq/enricher
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Enricher, the IP and URL data enhancer
+test_files: []