columbo 0.1.3 → 0.1.4

data/README.md

@@ -45,7 +45,7 @@ Check the Rack configuration:
 
 ## Disclaimer
 
-This is an alpha release and it is untested with Sinatra, it is tested with Rails 3 only.
+This is an alpha release, it is untested with Sinatra and Rails 3 only.
 UI to explore sessions will be completed later (ETA: 2013'Q2).
 
 ## Author

data/lib/columbo/capture.rb

@@ -10,9 +10,12 @@ module Columbo
 
     def initialize(app, opts={})
       @app = app
-      @capture    = opts[:capture]
-      @bench      = opts[:capture] && opts[:bench]
-      @mongo_uri  = opts[:mongo_uri]
+      # Options
+      @capture          = opts[:capture]
+      @bench            = opts[:capture] && opts[:bench]
+      @capture_crawlers = opts[:capture_crawlers]
+      @crawlers         = opts[:crawlers] || "(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg|bot|crawler|spider|robot|crawling|facebook|w3c|coccoc)"
+      @mongo_uri        = opts[:mongo_uri]
 
       Columbo.logger = opts[:logger] if opts[:logger]
 
@@ -43,7 +46,7 @@ module Columbo
 
         Thread.new do
           begin
-            @inspector.investigate env, status, headers, response, start_processing, stop_processing
+            @inspector.investigate env, status, headers, response, start_processing, stop_processing, @crawlers, @capture_crawlers
           rescue Exception => e
             log_error env, e
           end

data/lib/columbo/inspector.rb

@@ -10,11 +10,13 @@ module Columbo
       @mongo_uri = mongo_uri
     end
 
-    def investigate(env, status, headers, body, start, stop)
+    def investigate(env, status, headers, body, start, stop, crawlers, capture_crawlers)
       # Lazy connection to MongoDB
       client = Columbo::DbClient.new @mongo_uri
       # Normalise request from env
       request = Rack::Request.new(env)
+      rg = Regexp.new(crawlers, Regexp::IGNORECASE)
+      return if request.user_agent.match(rg) && !capture_crawlers
       html = ''
       body.each { |part| html += part }
       # Retrieve plain text body for full text search

data/lib/columbo/version.rb

@@ -1,3 +1,3 @@
 module Columbo
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: columbo
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-29 00:00:00.000000000 Z
+date: 2013-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack