httpbl 0.1.3 → 0.1.6

data/CHANGELOG

@@ -1 +1,3 @@
+v0.1.6. Adding memcache-client option to enable per-session lookups instead of per-request.  General refactoring.
+
 v0.1.3. First public test release, not ready for production

data/README

@@ -13,11 +13,12 @@ crawling, comment-spamming, dictionary attacks, and email-harvesting.
 		behavior after a customizable period of days.
 	* 	Identify common search engines by IP address (not User-Agent), and
 		disallow access to a specific subset.
+	*	Optionally use memcached to avoid repeated look-ups per client-session
 
 Installation
 ------------
 
-	gem install bpalmen-httpbl
+	gem install httpbl
 	
 Basic Usage
 ------------
@@ -52,11 +53,13 @@ To customize HttpBL's filtering behavior, use the available options:
 				:threat_level_threshold => 0,
 				:age_threshold => 5,
 				:blocked_search_engines => [0],
+				:memcached_server => "127.0.0.1:11211",
+				:memcached_options => {see: memcached-client documentation}
 
 Available Options:
 
 The following options (shown with default values) are available to 
-customize the particular types of suspicious activity you wish to thwart:
+customize the behavior of the httpbl middleware filter:
 
 	:deny_types => [1, 2, 4, 8, 16, 32, 64, 128]
 		
@@ -136,6 +139,23 @@ customize the particular types of suspicious activity you wish to thwart:
 			10: Cuil
 			11: InfoSeek
 	
+	:memcached_server => nil
+	:memcached_options => {}
+	
+		When using httpbl in a production environment, it is *strongly* recommended
+		that you configure httpbl to use memcached to temporarily store the blacklist
+		status of client ip addresses.  This greatly enhances the efficiency of the
+		filter because it need only look up each client ip address once per session,
+		instead of once per request.  It also reduces the potential burden of a 
+		popular web application that uses httpbl on project honeypot's api services.
+		
+		Simply set :memcached_server and :memcached_options according to the 
+		conventions of the memcache-client ruby library; for example:
+		:memcached_server => '127.0.0.1:11211', :memcached_options => {:namespace => 'my_app'}
+		
+		memcache-client is included in rails by default, but if you're using rack 
+		without rails, you will need to install and require the memcache-client gem. 
+	
 	:dns_timeout => 0.5
 	
 		DNS requests to the Http:BL service should NEVER take this long, but if

data/httpbl.gemspec

@@ -2,26 +2,25 @@
 
 Gem::Specification.new do |s|
   s.name = %q{httpbl}
-  s.version = "0.1.3"
+  s.version = "0.1.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brandon Palmen"]
-  s.date = %q{2009-03-22}
+  s.date = %q{2009-05-28}
   s.description = %q{A Rack middleware IP filter that uses Http:BL to exclude suspicious robots.}
   s.email = %q{}
   s.extra_rdoc_files = ["README", "lib/httpbl.rb", "CHANGELOG", "LICENSE"]
   s.files = ["README", "lib/httpbl.rb", "Rakefile", "httpbl.gemspec", "CHANGELOG", "LICENSE", "Manifest"]
-  s.has_rdoc = true
   s.homepage = %q{http://github.com/bpalmen/httpbl}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Httpbl", "--main", "README"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{httpbl}
-  s.rubygems_version = %q{1.3.1}
+  s.rubygems_version = %q{1.3.3}
   s.summary = %q{A Rack middleware IP filter that uses Http:BL to exclude suspicious robots.}
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 2
+    s.specification_version = 3
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<rack>, [">= 0"])

data/lib/httpbl.rb

@@ -8,12 +8,16 @@ class HttpBL
     @options = {:blocked_search_engines => [],
                 :age_threshold => 10,
                 :threat_level_threshold => 2,
-                # 8..128 aren't used as of 3/2009, but might be used in the future
-                :deny_types => [1, 2, 4, 8, 16, 32, 64, 128],
-                # DONT set this to 0
-                :dns_timeout => 0.5
+                :deny_types => [1, 2, 4, 8, 16, 32, 64, 128], # 8..128 aren't used as of 3/2009, but might be used in the future
+                :dns_timeout => 0.5,
+                :memcached_server => nil,
+		:memcached_options => {}
                 }.merge(options)
     raise "Missing :api_key for Http:BL middleware" unless @options[:api_key]
+    if @options[:memcached_server]
+      require 'memcache'
+      @cache = MemCache.new(@options[:memcached_server], @options[:memcached_options])
+    end
   end
   
   def call(env)
@@ -22,7 +26,7 @@ class HttpBL
   
   def _call(env)
     request = Rack::Request.new(env)
-    bl_status = resolve(request.ip)
+    bl_status = check(request.ip)
     if bl_status and blocked?(bl_status)
       [403, {"Content-Type" => "text/html"}, "<h1>403 Forbidden</h1> Request IP is listed as suspicious by <a href='http://projecthoneypot.org/ip_#{request.ip}'>Project Honeypot</a>"]
     else
@@ -31,10 +35,24 @@ class HttpBL
     
   end
   
+  def check(ip)
+    @cache ? cache_check(ip) : resolve(ip)
+  end
+  
+  def cache_check(ip)
+    cache = @cache.clone if @cache
+    unless response = cache.get("httpbl_#{ip}")
+      response = resolve(ip)
+      cache.set("httpbl_#{ip}", (response || "0.0.0.0"), 1.hour)
+    end
+    return response
+  end
+  
   def resolve(ip)
     query = @options[:api_key] + '.' + ip.split('.').reverse.join('.') + '.dnsbl.httpbl.org'
     Timeout::timeout(@options[:dns_timeout]) do
-       Resolv::DNS.new.getaddress(query).to_s rescue nil
+       Resolv::DNS.new.getaddress(query).to_s rescue false
+	puts "resolving"
     end
     rescue Timeout::Error, Errno::ECONNREFUSED
   end
@@ -43,14 +61,11 @@ class HttpBL
     response = response.split('.').collect!(&:to_i)
     if response[0] == 127 
       if response[3] == 0
-        @blocked = true if @options[:blocked_search_engines].include? response[2]
+        @blocked = @options[:blocked_search_engines].include?(response[2])
       else 
-        @age = true if response[1] < @options[:age_threshold]
-        @threat = true if response[2] > @options[:threat_level_threshold]
-        @options[:deny_types].each do |key|
-          @deny = true if response[3] & key == key  
-        end
-        @blocked = true if @deny and @threat and @age
+        @blocked = @options[:deny_types].collect{|key| response[3] & key == key }.any?
+        @blocked = @blocked and response[2] > @options[:threat_level_threshold] 
+        @blocked = @blocked and response[1] < @options[:age_threshold]
       end
     end
     return @blocked

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: httpbl
 version: !ruby/object:Gem::Version 
-  version: 0.1.3
+  version: 0.1.6
 platform: ruby
 authors: 
 - Brandon Palmen
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-03-22 00:00:00 -04:00
+date: 2009-05-28 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -43,6 +43,8 @@ files:
 - Manifest
 has_rdoc: true
 homepage: http://github.com/bpalmen/httpbl
+licenses: []
+
 post_install_message: 
 rdoc_options: 
 - --line-numbers
@@ -68,9 +70,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: httpbl
-rubygems_version: 1.3.1
+rubygems_version: 1.3.3
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: A Rack middleware IP filter that uses Http:BL to exclude suspicious robots.
 test_files: []