cobweb 1.0.12 → 1.0.15

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODVkZmRkYjU3NjVkN2U2NjEzYmNlZDQxYTI4MDc3N2Y5MzBjZjhkOA==
+  data.tar.gz: !binary |-
+    MGU3M2EzNmJiMTk4NzdjZTY0MjJiY2M1YTQ5ZDcyOGY2ZjEzNzU0OQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MzI0NTc1YTM4ODYxNjVhZWE1ZDEwZjI5MWEzYzc3MmYwMzExNDdkM2U0OThl
+    MTRhOTA3ZGRmN2RjYWUzODlmYjgwM2U5Mzg1ZmU1OWY0YWFkMWU1ODYyMTU1
+    Y2ViYjNjZmZiOTExNWU0MDFiN2QzODExMzc3NTVhNjRmNGJjNmE=
+  data.tar.gz: !binary |-
+    NTdkZWJhNTgzYzU5YjRjYzI4ZmYxMDE2NGNhZTkxZWFkMTBlZTU5MWQ0YjUw
+    YzAyOGQ0Y2Y1NjI1OGUyY2MxM2M0NGU0MTZhMWIzMDcwZTA3MmZhMTg4NmUx
+    YzE5NWZhZjljZTM5NzI5MWFhODQ1MDI1YjI3Zjc0MjlhODdhNjE=

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v1.0.12
+h1. Cobweb v1.0.15
 
 "@cobweb_gem":https://twitter.com/cobweb_gem
 !https://badge.fury.io/rb/cobweb.png!:http://badge.fury.io/rb/cobweb
@@ -106,6 +106,7 @@ Creates a new crawler object based on a base_url
     ** :crawl_id                      - the id used internally for identifying the crawl.  Can be used by the processing job to seperate crawls
     ** :internal_urls                 - an array of urls with * wildcards that represent urls internal to the site (ie pages within the same domain)
     ** :external_urls                 - an array of urls with * wildcards that represent urls external to the site (overrides internal_urls)
+    ** :seed_urls                     - an array of urls that are put into the queue regardless of any other setting, combine with {:external_urls => "*"} to limit to seed urls
     ** :obey_robots                   - boolean determining if robots.txt should be honoured. (default: false)
     ** :user_agent                    - user agent string to match in robots.txt (not sent as user_agent of requests yet) (default: cobweb)
     ** :crawl_limit_by_page           - sets the crawl counter to only use html page types when counting objects crawled

data/lib/cobweb.rb

@@ -49,6 +49,7 @@ class Cobweb
     default_redis_options_to                  Hash.new
     default_internal_urls_to                  []
     default_external_urls_to                  []
+    default_seed_urls_to                  []
     default_first_page_redirect_internal_to   true
     default_text_mime_types_to                ["text/*", "application/xhtml+xml"]
     default_obey_robots_to                    false
@@ -80,6 +81,8 @@ class Cobweb
     @redis.hset "statistics", "queued_at", DateTime.now
     @redis.set("crawl-counter", 0)
     @redis.set("queue-counter", 1)
+
+    @options[:seed_urls].map{|link| @redis.sadd "queued", link }
     
     @stats = Stats.new(request)
     @stats.start_crawl(request)
@@ -138,10 +141,10 @@ class Cobweb
     # check if it has already been cached
     if ((@options[:cache_type] == :crawl_based && redis.get(unique_id)) || (@options[:cache_type] == :full && full_redis.get(unique_id))) && @options[:cache]
       if @options[:cache_type] == :crawl_based 
-        puts "Cache hit for #{url}" unless @options[:quiet]
+        puts "Cache hit in crawl for #{url}" #unless @options[:quiet]
         content = HashUtil.deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
       else
-        puts "Cache hit for #{url}" unless @options[:quiet]
+        puts "Cache hit for #{url}" #unless @options[:quiet]
         content = HashUtil.deep_symbolize_keys(Marshal.load(full_redis.get(unique_id)))
       end
     else

data/lib/cobweb_crawler.rb

@@ -23,6 +23,8 @@ class CobwebCrawler
     @redis = Redis::Namespace.new("cobweb-#{Cobweb.version}-#{@crawl_id}", :redis => Redis.new(@options[:redis_options]))
     @options[:internal_urls] = [] if @options[:internal_urls].nil?
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
+    @options[:seed_urls] = [] if @options[:seed_urls].nil?
+    @options[:seed_urls].map{|link| @redis.sadd "queued", link }
 
     @options[:crawl_linked_external] = false unless @options.has_key? :crawl_linked_external
     

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "1.0.12"
+    "1.0.15"
   end
 
 end

data/lib/server.rb

@@ -7,6 +7,7 @@ class Server < Sinatra::Base
   set :root, File.dirname(__FILE__)
   set :views, settings.root + '/../views'
   set :public_folder, settings.root + '/../public'
+  set :bind, '0.0.0.0'
   enable :static
   
   # Sinatra Dashboard
@@ -26,6 +27,7 @@ class Server < Sinatra::Base
           :minute_totals => redis.hgetall("minute_totals"),
           })
         @crawls << stats
+        @crawls.sort!{|a,b| b[:statistics][:crawl_started_at] <=> a[:statistics][:crawl_started_at]}
       end
     end
     
@@ -63,10 +65,8 @@ class Server < Sinatra::Base
         :pages_count => HashUtil.deep_symbolize_keys(redis.hgetall("pages_count")),
         :assets_count => HashUtil.deep_symbolize_keys(redis.hgetall("assets_count"))
     }
-    (1..30).each do |minutes|
-      date = (DateTime.now.new_offset(0) - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
-    end
-    
+
+    @dates = (1..30).to_a.reverse.map{|minutes| [(DateTime.now - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym, minutes] }
     haml :statistics
   end
   

data/lib/stats.rb

@@ -20,6 +20,7 @@ class Stats
         @redis.hset "crawl_details", key, options[key].to_s
       end
     end
+    @redis.hset "statistics", "crawl_started_at", DateTime.now
     @redis.hset "statistics", "current_status", CobwebCrawlHelper::STARTING
   end
   
@@ -31,6 +32,7 @@ class Stats
     else
       @redis.hset "statistics", "current_status", CobwebCrawlHelper::FINISHED
     end
+    @redis.hset "statistics", "crawl_finished_at", DateTime.now
     #@redis.del "crawl_details"
   end
   

data/public/js/jquery.flot.js

@@ -56,7 +56,7 @@
                 xaxis: {
                     show: null, // null = auto-detect, true = always, false = never
                     position: "bottom", // or "top"
-                    mode: null, // null or "time"
+                    mode: "time", // null or "time"
                     color: null, // base color, labels, ticks
                     tickColor: null, // possibly different color of ticks, e.g. "rgba(0,0,0,0.15)"
                     transform: null, // null or f: number -> number to transform axis
@@ -77,7 +77,7 @@
                     tickSize: null, // number or [number, "unit"]
                     minTickSize: null, // number or [number, "unit"]
                     monthNames: null, // list of names of months
-                    timeformat: null, // format string to use
+                    timeformat: "%H:%M", // format string to use
                     twelveHourClock: false // 12 or 24 time in time mode
                 },
                 yaxis: {

data/spec/http_stubs.rb

@@ -21,75 +21,75 @@ module HttpStubs
                         :"X-XSS-Protection" => "1; mode=block"}
     
 
-    @mock_http_client = mock(Net::HTTP)
-    @mock_http_request = mock(Net::HTTPRequest)
-    @mock_http_robot_request = mock(Net::HTTPRequest)
-    @mock_http_redirect_request = mock(Net::HTTPRequest)
-    @mock_http_redirect_request2 = mock(Net::HTTPRequest)
+    @mock_http_client = double(Net::HTTP)
+    @mock_http_request = double(Net::HTTPRequest)
+    @mock_http_robot_request = double(Net::HTTPRequest)
+    @mock_http_redirect_request = double(Net::HTTPRequest)
+    @mock_http_redirect_request2 = double(Net::HTTPRequest)
     
-    @mock_http_response = mock(Net::HTTPResponse)
-    @mock_http_robot_response = mock(Net::HTTPResponse)
-    @mock_http_redirect_response = mock(Net::HTTPRedirection)
-    @mock_http_redirect_response2 = mock(Net::HTTPRedirection)
-    @mock_http_get = mock(Net::HTTP::Get)
+    @mock_http_response = double(Net::HTTPResponse)
+    @mock_http_robot_response = double(Net::HTTPResponse)
+    @mock_http_redirect_response = double(Net::HTTPRedirection)
+    @mock_http_redirect_response2 = double(Net::HTTPRedirection)
+    @mock_http_get = double(Net::HTTP::Get)
     
-    Net::HTTP.stub!(:new).and_return(@mock_http_client)
-    Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect.html", an_instance_of(Hash)).and_return(@mock_http_redirect_request)
-    Net::HTTP::Get.stub!(:new).with("/robots.txt", an_instance_of(Hash)).and_return(@mock_http_robot_request)
-    Net::HTTP::Get.stub!(:new).with("/redirect2.html", an_instance_of(Hash)).and_return(@mock_http_redirect_request2)
-    Net::HTTP::Get.stub!(:new).with("/redirected.html", an_instance_of(Hash)).and_return(@mock_http_request)
+    Net::HTTP.stub(:new).and_return(@mock_http_client)
+    Net::HTTP::Get.stub(:new).and_return(@mock_http_request)
+    Net::HTTP::Get.stub(:new).with("/redirect.html", an_instance_of(Hash)).and_return(@mock_http_redirect_request)
+    Net::HTTP::Get.stub(:new).with("/robots.txt", an_instance_of(Hash)).and_return(@mock_http_robot_request)
+    Net::HTTP::Get.stub(:new).with("/redirect2.html", an_instance_of(Hash)).and_return(@mock_http_redirect_request2)
+    Net::HTTP::Get.stub(:new).with("/redirected.html", an_instance_of(Hash)).and_return(@mock_http_request)
 
-    Net::HTTP::Head.stub!(:new).and_return(@mock_http_request)
+    Net::HTTP::Head.stub(:new).and_return(@mock_http_request)
 
-    @mock_http_client.stub!(:request).with(@mock_http_request).and_return(@mock_http_response)
-    @mock_http_client.stub!(:request).with(@mock_http_robot_request).and_return(@mock_http_robot_response)
-    @mock_http_client.stub!(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
-    @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)
-    @mock_http_client.stub!(:read_timeout=).and_return(nil)      
-    @mock_http_client.stub!(:open_timeout=).and_return(nil)      
-    @mock_http_client.stub!(:start).and_return(@mock_http_response)
-    @mock_http_client.stub!(:address).and_return("www.baseurl.com")
-    @mock_http_client.stub!(:port).and_return("80")
+    @mock_http_client.stub(:request).with(@mock_http_request).and_return(@mock_http_response)
+    @mock_http_client.stub(:request).with(@mock_http_robot_request).and_return(@mock_http_robot_response)
+    @mock_http_client.stub(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
+    @mock_http_client.stub(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)
+    @mock_http_client.stub(:read_timeout=).and_return(nil)      
+    @mock_http_client.stub(:open_timeout=).and_return(nil)      
+    @mock_http_client.stub(:start).and_return(@mock_http_response)
+    @mock_http_client.stub(:address).and_return("www.baseurl.com")
+    @mock_http_client.stub(:port).and_return("80")
     
-    @mock_http_robot_response.stub!(:code).and_return(200)
-    @mock_http_robot_response.stub!(:body).and_return(File.open(File.dirname(__FILE__) + '/../spec/samples/robots.txt', "r").read)
-    @mock_http_robot_response.stub!(:content_type).and_return("text/plain")
-    @mock_http_robot_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-    @mock_http_robot_response.stub!(:[]).with("location").and_return(@default_headers["location"])
-    @mock_http_robot_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
-    @mock_http_robot_response.stub!(:content_length).and_return(1024)
-    @mock_http_robot_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
-    @mock_http_robot_response.stub!(:to_hash).and_return(@default_headers)
+    @mock_http_robot_response.stub(:code).and_return(200)
+    @mock_http_robot_response.stub(:body).and_return(File.open(File.dirname(__FILE__) + '/../spec/samples/robots.txt', "r").read)
+    @mock_http_robot_response.stub(:content_type).and_return("text/plain")
+    @mock_http_robot_response.stub(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_robot_response.stub(:[]).with("location").and_return(@default_headers["location"])
+    @mock_http_robot_response.stub(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_robot_response.stub(:content_length).and_return(1024)
+    @mock_http_robot_response.stub(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
+    @mock_http_robot_response.stub(:to_hash).and_return(@default_headers)
     
-    @mock_http_response.stub!(:code).and_return(200)
-    @mock_http_response.stub!(:content_type).and_return("text/html")
-    @mock_http_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-    @mock_http_response.stub!(:[]).with("location").and_return(@default_headers["location"])
-    @mock_http_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
-    @mock_http_response.stub!(:content_length).and_return(1024)
-    @mock_http_response.stub!(:body).and_return("asdf")
-    @mock_http_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
-    @mock_http_response.stub!(:to_hash).and_return(@default_headers)
+    @mock_http_response.stub(:code).and_return(200)
+    @mock_http_response.stub(:content_type).and_return("text/html")
+    @mock_http_response.stub(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_response.stub(:[]).with("location").and_return(@default_headers["location"])
+    @mock_http_response.stub(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_response.stub(:content_length).and_return(1024)
+    @mock_http_response.stub(:body).and_return("asdf")
+    @mock_http_response.stub(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
+    @mock_http_response.stub(:to_hash).and_return(@default_headers)
     
-    @mock_http_redirect_response.stub!(:code).and_return(301)
-    @mock_http_redirect_response.stub!(:content_type).and_return("text/html")
-    @mock_http_redirect_response.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-    @mock_http_redirect_response.stub!(:[]).with("location").and_return("http://redirected-to.com/redirect2.html")
-    @mock_http_redirect_response.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
-    @mock_http_redirect_response.stub!(:content_length).and_return(2048)
-    @mock_http_redirect_response.stub!(:body).and_return("redirected body")
-    @mock_http_redirect_response.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
-    @mock_http_redirect_response.stub!(:to_hash).and_return(@default_headers)
+    @mock_http_redirect_response.stub(:code).and_return(301)
+    @mock_http_redirect_response.stub(:content_type).and_return("text/html")
+    @mock_http_redirect_response.stub(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_redirect_response.stub(:[]).with("location").and_return("http://redirected-to.com/redirect2.html")
+    @mock_http_redirect_response.stub(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_redirect_response.stub(:content_length).and_return(2048)
+    @mock_http_redirect_response.stub(:body).and_return("redirected body")
+    @mock_http_redirect_response.stub(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
+    @mock_http_redirect_response.stub(:to_hash).and_return(@default_headers)
     
-    @mock_http_redirect_response2.stub!(:code).and_return(301)
-    @mock_http_redirect_response2.stub!(:content_type).and_return("text/html")
-    @mock_http_redirect_response2.stub!(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
-    @mock_http_redirect_response2.stub!(:[]).with("location").and_return("http://redirected-to.com/redirected.html")
-    @mock_http_redirect_response2.stub!(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
-    @mock_http_redirect_response2.stub!(:content_length).and_return(2048)
-    @mock_http_redirect_response2.stub!(:body).and_return("redirected body")
-    @mock_http_redirect_response2.stub!(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
-    @mock_http_redirect_response2.stub!(:to_hash).and_return(@default_headers)
+    @mock_http_redirect_response2.stub(:code).and_return(301)
+    @mock_http_redirect_response2.stub(:content_type).and_return("text/html")
+    @mock_http_redirect_response2.stub(:[]).with("Content-Type").and_return(@default_headers["Content-Type"])
+    @mock_http_redirect_response2.stub(:[]).with("location").and_return("http://redirected-to.com/redirected.html")
+    @mock_http_redirect_response2.stub(:[]).with("Content-Encoding").and_return(@default_headers["Content-Encoding"])
+    @mock_http_redirect_response2.stub(:content_length).and_return(2048)
+    @mock_http_redirect_response2.stub(:body).and_return("redirected body")
+    @mock_http_redirect_response2.stub(:get_fields).with('set-cookie').and_return(["session=al98axx; expires=Fri, 31-Dec-1999 23:58:23", "query=rubyscript; expires=Fri, 31-Dec-1999 23:58:23"])
+    @mock_http_redirect_response2.stub(:to_hash).and_return(@default_headers)
   end
 end

data/views/statistics.haml

@@ -30,7 +30,7 @@
         %h2 Pages and Assets
         %span.hide
       .content
-        .caption Counts of assets and pages that have been crawled
+        .caption Counts of assets and pages that have been crawled over the last 30 mins
         %table.statics{:style => "width : 100%;"}
           %thead
             %tr
@@ -39,8 +39,7 @@
               %th{:style => "color: #006ba0;"} Assets
 
           %tbody
-            - (1..30).each do |minutes|
-              - date = (DateTime.now.new_offset(0) - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
+            - @dates.each do |date, minutes|
               %tr
                 %th= minutes
                 %td= @crawl[:pages_count][date]
@@ -88,7 +87,7 @@
 
           %tbody
             - (1..30).each do |minutes|
-              - date = (DateTime.now.new_offset(0) - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
+              - date = (DateTime.now - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
               %tr
                 %th= minutes
                 %td= @crawl[:status_200_count][date]
@@ -113,7 +112,7 @@
 
           %tbody
             - (1..30).each do |minutes|
-              - date = (DateTime.now.new_offset(0) - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
+              - date = (DateTime.now - (minutes/1440.0)).strftime("%Y-%m-%d %H:%M").to_sym
               %tr
                 %th= minutes
                 %td= @crawl[:mime_text_count][date]

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.12
-  prerelease: 
+  version: 1.0.15
 platform: ruby
 authors:
 - Stewart McKee
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-15 00:00:00.000000000 Z
+date: 2013-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -110,7 +97,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: thin
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -118,7 +104,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -126,7 +111,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: haml
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -134,7 +118,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -142,7 +125,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -150,7 +132,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -158,7 +139,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -166,7 +146,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -174,7 +153,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -182,7 +160,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -562,27 +539,26 @@ files:
 - README.textile
 homepage: http://github.com/stewartmckee/cobweb
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites faster than multi-threaded crawlers.  It is also a standalone
   crawler that has a sophisticated statistics monitoring interface to monitor the