cobweb 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b21efd1b03e9f4515045bdd69439fc8a7a4bf3f5aa896edebdbccd12d421d51
-  data.tar.gz: 4744738dfb7eaca5c5b6c1cabdebea8899e0f18cadb5532f1bb43870e98ff4e2
+  metadata.gz: 2fa16c36a203417d0f90f884634267227698d83839580c59982d15b01bb1f495
+  data.tar.gz: a262f311869ccd9696fd377c1bd2de04ab32f19b1709df1d3321b80e9d4e8517
 SHA512:
-  metadata.gz: 55b7f9878fcf6c3d97bd935fa59e61c54d745dd37cc75405e9af0e787d56e6543d79a2dedfca43912ed791fd59b51193b12c58722be9ce8c5d65428b6ea0af48
-  data.tar.gz: 8f767e79d76787cd84edbecd967e3136615a32723210a1abe149f8c126e4c8e1c6b3608f631f4ea7d3ad4fe66aa39702b18f73e28c0091ab02b2defdb28868d2
+  metadata.gz: 1b32dce38f7563f75dce48d4e87dfa083c918b1f3702a2f95159939fb09a30742ea27be5dd3e1551451f268df86cbbf9614603f030cfc114cc421fe17cda7950
+  data.tar.gz: dc8dfcdde6e157b73d4ff785d95afbb2d71ff3180169685762683a2b6aeeb37d1b47743608f76e3d2b944f37b12a3cab0cfd0a4bc1bb05c902d447b0f648709e

data/README.textile

@@ -189,6 +189,8 @@ bc. crawl = CobwebCrawlHelper.new(options)
 
 h2. Contributing/Testing
 
+  <p>Firstly, you could <a href="https://www.buymeacoffee.com/Z2yRGl3CX" target="_blank"><img src="https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png" alt="Buy Me A Coffee" style="height: auto !important;width: auto !important;" ></a></p>
+
   Feel free to contribute small or large bits of code, just please make sure that there are rspec test for the features your submitting.  We also test on travis at http://travis-ci.org/#!/stewartmckee/cobweb if you want to see the state of the project.
 
     Continuous integration testing is performed by the excellent Travis: http://travis-ci.org/#!/stewartmckee/cobweb

data/lib/cobweb.rb

@@ -77,8 +77,17 @@ class Cobweb
     if @options[:internal_urls].nil? || @options[:internal_urls].empty?
       uri = Addressable::URI.parse(base_url)
       @options[:internal_urls] = []
-      @options[:internal_urls] << [uri.scheme, "://", uri.host, "/*"].join
-      @options[:internal_urls] << [uri.scheme, "://", uri.host, ":", uri.inferred_port, "/*"].join
+
+      if @options[:treat_https_as_http_to]
+        @options[:internal_urls] << ["http://", uri.host, "/*"].join
+        @options[:internal_urls] << ["http://", uri.host, ":", uri.inferred_port, "/*"].join
+        @options[:internal_urls] << ["https://", uri.host, "/*"].join
+        @options[:internal_urls] << ["https://", uri.host, ":", uri.inferred_port, "/*"].join
+      else
+        @options[:internal_urls] << [uri.scheme, "://", uri.host, "/*"].join
+        @options[:internal_urls] << [uri.scheme, "://", uri.host, ":", uri.inferred_port, "/*"].join
+      end
+
     end
 
     request.merge!(@options)
@@ -86,7 +95,10 @@ class Cobweb
     @redis.set("original_base_url", base_url)
     @redis.hset "statistics", "queued_at", DateTime.now
     @redis.set("crawl-counter", 0)
-    @redis.set("queue-counter", 1)
+    queue_counter = @options[:seed_urls].count + 1
+    puts "queue_counter being init to #{queue_counter}"
+    @redis.set("queue-counter", queue_counter)
+
 
     @options[:seed_urls].map{|link| @redis.sadd "queued", link }
 
@@ -99,6 +111,12 @@ class Cobweb
       Resque.enqueue(CrawlJob, request)
     elsif @options[:queue_system] == :sidekiq
       CrawlWorker.perform_async(request)
+      @options[:seed_urls].map{|url| 
+        new_request = request.clone
+        new_request[:url] = url
+        CrawlWorker.perform_async(new_request)
+      }
+
     else
       raise "Unknown queue system: #{content_request[:queue_system]}"
     end
@@ -170,7 +188,7 @@ class Cobweb
       begin
         puts "Retrieving #{uri}... " unless @options[:quiet]
         request_options={}
-        request_options['Cookie']= options[:cookies] if options[:cookies]
+        request_options['Cookie']= options[:cookies].map{|k,v| [k,v].join("=") }.join("&") if options[:cookies]
         request_options['User-Agent']= options[:user_agent] if options.has_key?(:user_agent)
 
         request = Net::HTTP::Get.new uri.request_uri, request_options
@@ -185,6 +203,8 @@ class Cobweb
 
         response = @http.request request
 
+        cookies = Hash[get_cookies(response).to_s.split("; ").map{|s| [CGI.unescape(s.split("=")[0]), s.split("=")[1]]}].merge(options[:cookies] || {})
+
         if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
 
           # get location to redirect to
@@ -195,8 +215,7 @@ class Cobweb
           redirect_limit = redirect_limit - 1
 
           raise RedirectError, "Redirect Limit reached" if redirect_limit == 0
-          cookies = get_cookies(response)
-
+          
           # get the content from redirect location
           content = get(uri, options.merge(:redirect_limit => redirect_limit, :cookies => cookies))
 
@@ -213,6 +232,7 @@ class Cobweb
           # create the content container
           content[:url] = uri.to_s
           content[:status_code] = response.code.to_i
+          content[:cookies] = cookies
           content[:mime_type] = ""
           content[:mime_type] = response.content_type.split(";")[0].strip unless response.content_type.nil?
           if !response["Content-Type"].nil? && response["Content-Type"].include?(";")

data/lib/cobweb_links.rb

@@ -28,16 +28,16 @@ class CobwebLinks
   
   # Returns true if the link is matched to an internal_url and not matched to an external_url
   def internal?(link)
-    !@internal_patterns.select{|pattern| link.match(pattern)}.empty? && @external_patterns.select{|pattern| link.match(pattern)}.empty?
+    @internal_patterns.any?{|pattern| link.match(pattern)} && !@external_patterns.any?{|pattern| link.match(pattern)}
   end
   
   # Returns true if the link is matched to an external_url or not matched to an internal_url
   def external?(link)
-    @internal_patterns.select{|pattern| link.match(pattern)}.empty? || !@external_patterns.select{|pattern| link.match(pattern)}.empty?
+    !@internal_patterns.any?{|pattern| link.match(pattern)} || @external_patterns.any?{|pattern| link.match(pattern)}
   end
 
   def matches_external?(link)
-    !@external_patterns.select{|pattern| link.match(pattern)}.empty?
+    @external_patterns.any?{|pattern| link.match(pattern)}
   end
   
 end

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
 
   # Returns a string of the current version
   def self.version
-    "1.2.0"
+    "1.2.1"
   end
 
 end

data/lib/crawl.rb

@@ -70,6 +70,8 @@ module CobwebModule
               @redis.sadd("currently_running", @options[:url])
               @stats.update_status("Retrieving #{@options[:url]}...")
               @content = Cobweb.new(@options).get(@options[:url], @options)
+              @cookies = @content[:cookies]
+
               update_counters
 
               if @options[:url] == @redis.get("original_base_url")
@@ -117,7 +119,6 @@ module CobwebModule
         document_links = ContentLinkParser.new(@options[:url], content.body, @options).all_links(:valid_schemes => [:http, :https])
         #get rid of duplicate links in the same page.
         document_links.uniq!
-        
         # select the link if its internal
         internal_links = document_links.select{ |link| @cobweb_links.internal?(link) }
 
@@ -125,24 +126,16 @@ module CobwebModule
         if @options[:treat_https_as_http]
           internal_links.map!{|link| link.gsub(/^https/, "http")}
         end
-
         # reject the link if we've crawled it or queued it
         internal_links.reject! { |link| already_handled?(link)}
 
-        lock("internal-links") do
-          internal_links.each do |link|
-            if within_queue_limits? && !already_handled?(link)
-              if status != CobwebCrawlHelper::CANCELLED
-                yield link if block_given?
-                unless link.nil?
-                  @redis.sadd "queued", link
-                  increment_queue_counter
-                end
-              else
-                debug_puts "Cannot enqueue new content as crawl has been cancelled."
-              end
-            end
+        if status != CobwebCrawlHelper::CANCELLED && internal_links.present?
+          lock("internal-links") do
+            @redis.sadd "queued", internal_links
+            @redis.incrby "queue-counter", internal_links.count
+
           end
+          internal_links.map{|link| yield(link) if block_given? }
         end
 
         if @options[:store_inbound_links]
@@ -154,6 +147,10 @@ module CobwebModule
       end
     end
 
+    def cookies
+      @cookies
+    end
+
     def content
       raise "Content is not available" if @content.nil?
       CobwebModule::CrawlObject.new(@content, @options)
@@ -182,7 +179,7 @@ module CobwebModule
     end
 
     def to_be_processed?
-      !finished? && within_process_limits? && !@redis.sismember("queued", @options[:url])
+      !finished? && within_process_limits?
     end
 
     def process(&block)
@@ -275,7 +272,7 @@ module CobwebModule
       debug_puts "LOCK:#{key}:#{set_nx}"
       while !set_nx
         debug_puts "===== WAITING FOR LOCK [#{key}] ====="
-        sleep 0.01
+        sleep 0.5
         set_nx = @redis.setnx("#{key}_lock", "locked")
       end
 
@@ -285,7 +282,7 @@ module CobwebModule
         result = yield
       ensure
         @redis.del("#{key}_lock")
-        #debug_puts "LOCK RELEASED [#{key}]"
+        debug_puts "LOCK RELEASED [#{key}]"
       end
       result
     end

data/lib/crawl_worker.rb

@@ -20,6 +20,8 @@ class CrawlWorker
     # setup the crawl class to manage the crawl of this object
     @crawl = CobwebModule::Crawl.new(content_request)
 
+    content_request.merge!(:cookies => @crawl.cookies)
+    
     # update the counters and then perform the get, returns false if we are outwith limits
     if @crawl.retrieve
 
@@ -28,13 +30,12 @@ class CrawlWorker
 
         # extract links from content and process them if we are still within queue limits (block will not run if we are outwith limits)
         @crawl.process_links do |link|
-          @crawl.lock("queue_links") do
-            if @crawl.within_crawl_limits? && !@crawl.already_handled?(link)
+          puts "Looking to add #{link}.."
+          # @crawl.lock("queue_links") do
               # enqueue the links to sidekiq
-              @crawl.debug_puts "QUEUED LINK: #{link}"
-              enqueue_content(content_request, link)
-            end
-          end
+            @crawl.debug_puts "QUEUED LINK: #{link}"
+            enqueue_content(content_request, link)
+          # end
         end
 
         if @crawl.to_be_processed?

data/spec/cobweb/cobweb_crawl_spec.rb

@@ -28,22 +28,22 @@ describe CobwebModule::Crawl, :local_only => true do
     end
     it "should connect to the local redis" do
       if @remote
-        @local.redis.exists("test_redis").should be_false
+        @local.redis.exists("test_redis").should be_falsey
         @local.redis.set("test_redis", 1)
-        @local.redis.exists("test_redis").should be_true
+        @local.redis.exists("test_redis").should be_truthy
 
-        @remote.redis.exists("test_redis").should be_false
+        @remote.redis.exists("test_redis").should be_falsey
       else
         puts "WARNING: can't connect to remote redis"
       end
     end
     it "should connect to the remote redis" do
       if @remote
-        @remote.redis.exists("test_redis").should be_false
+        @remote.redis.exists("test_redis").should be_falsey
         @remote.redis.set("test_redis", 1)
-        @remote.redis.exists("test_redis").should be_true
+        @remote.redis.exists("test_redis").should be_truthy
         
-        @local.redis.exists("test_redis").should be_false
+        @local.redis.exists("test_redis").should be_falsey
       else
         puts "WARNING: can't connect to remote redis"
       end

data/spec/cobweb/cobweb_links_spec.rb

@@ -41,69 +41,69 @@ describe CobwebLinks do
   describe "internal and external links" do
     it "should only return internal links" do
       cobweb_links = CobwebLinks.new(:internal_urls => ["http://domain_one.com/"], :external_urls => ["http://domain_two.com/"])
-      cobweb_links.internal?("http://domain_one.com/pageone.html").should be_true
-      cobweb_links.internal?("http://domain_one.com/pagetwo.html").should be_true
+      cobweb_links.internal?("http://domain_one.com/pageone.html").should be_truthy
+      cobweb_links.internal?("http://domain_one.com/pagetwo.html").should be_truthy
     end
     it "should not return external links" do
       cobweb_links = CobwebLinks.new(:internal_urls => ["http://domain_one.com/"], :external_urls => ["http://domain_two.com/"])
-      cobweb_links.external?("http://domain_one.com/pageone.html").should be_false
-      cobweb_links.external?("http://domain_two.com/pageone.html").should be_true      
-      cobweb_links.external?("http://external.com/pageone.html").should be_true      
+      cobweb_links.external?("http://domain_one.com/pageone.html").should be_falsey
+      cobweb_links.external?("http://domain_two.com/pageone.html").should be_truthy      
+      cobweb_links.external?("http://external.com/pageone.html").should be_truthy      
     end
     it "should override internal links with external links" do
       cobweb_links = CobwebLinks.new(:internal_urls => ["http://domain_one.com/"], :external_urls => ["http://domain_one.com/blog"])
-      cobweb_links.internal?("http://domain_one.com/pageone.html").should be_true
-      cobweb_links.external?("http://domain_one.com/pageone.html").should be_false
-      cobweb_links.internal?("http://domain_one.com/blog/pageone.html").should be_false
-      cobweb_links.external?("http://domain_one.com/blog/pageone.html").should be_true
-      cobweb_links.internal?("http://domain_two.com/blog/pageone.html").should be_false
-      cobweb_links.external?("http://domain_two.com/blog/pageone.html").should be_true
+      cobweb_links.internal?("http://domain_one.com/pageone.html").should be_truthy
+      cobweb_links.external?("http://domain_one.com/pageone.html").should be_falsey
+      cobweb_links.internal?("http://domain_one.com/blog/pageone.html").should be_falsey
+      cobweb_links.external?("http://domain_one.com/blog/pageone.html").should be_truthy
+      cobweb_links.internal?("http://domain_two.com/blog/pageone.html").should be_falsey
+      cobweb_links.external?("http://domain_two.com/blog/pageone.html").should be_truthy
     end
   end
   it "should only match from beginning of url" do
     cobweb_links = CobwebLinks.new(:internal_urls => ["http://www.domain_one.com/"], :external_urls => ["http://www.domain_two.com/"])
-    cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_true
-    cobweb_links.internal?("http://www.domain_two.com/pageone.html").should be_false
-    cobweb_links.internal?("http://www.domain_one.com/pageone.html?url=http://www.domain_two.com/pageone.html").should be_true
-    cobweb_links.internal?("http://www.domain_two.com/pageone.html?url=http://www.domain_one.com/pageone.html").should be_false
+    cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_truthy
+    cobweb_links.internal?("http://www.domain_two.com/pageone.html").should be_falsey
+    cobweb_links.internal?("http://www.domain_one.com/pageone.html?url=http://www.domain_two.com/pageone.html").should be_truthy
+    cobweb_links.internal?("http://www.domain_two.com/pageone.html?url=http://www.domain_one.com/pageone.html").should be_falsey
   end
   
   describe "using wildcards" do
     it "should match internal links with wildcards" do
       cobweb_links = CobwebLinks.new(:internal_urls => ["http://*.domain_one.com/"], :external_urls => ["http://blog.domain_one.com/"])
-      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_true
-      cobweb_links.internal?("http://images.domain_one.com/logo.png").should be_true      
-      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_false
+      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_truthy
+      cobweb_links.internal?("http://images.domain_one.com/logo.png").should be_truthy      
+      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_falsey
     end
     it "should match external links with wildcards" do
       cobweb_links = CobwebLinks.new(:internal_urls => ["http://www.domain_one.com/"], :external_urls => ["http://*.domain_one.com/"])
-      cobweb_links.external?("http://www.domain_one.com/pageone.html").should be_true
-      cobweb_links.external?("http://images.domain_one.com/logo.png").should be_true
-      cobweb_links.external?("http://blog.domain_one.com/pageone.html").should be_true
+      cobweb_links.external?("http://www.domain_one.com/pageone.html").should be_truthy
+      cobweb_links.external?("http://images.domain_one.com/logo.png").should be_truthy
+      cobweb_links.external?("http://blog.domain_one.com/pageone.html").should be_truthy
     end
     it "should match external links with querystring parameters" do
       cobweb_links = CobwebLinks.new(:internal_urls => ["http://www.ford.com/"], :external_urls => ["http://*.ford.com/*?*view=print"])
-      cobweb_links.external?("http://corporate.ford.com/news-center/press-releases-detail/pr-doug-scott2658-marketing-manager-31039?view=print").should be_true
-      cobweb_links.internal?("http://corporate.ford.com/news-center/press-releases-detail/pr-doug-scott2658-marketing-manager-31039?view=print").should be_false
-      cobweb_links.external?("http://corporate.ford.com/news-center/view=print/pr-doug-scott2658-marketing-manager-31039").should be_true
+      cobweb_links.external?("http://corporate.ford.com/news-center/press-releases-detail/pr-doug-scott2658-marketing-manager-31039?view=print").should be_truthy
+      cobweb_links.internal?("http://corporate.ford.com/news-center/press-releases-detail/pr-doug-scott2658-marketing-manager-31039?view=print").should be_falsey
+      cobweb_links.external?("http://corporate.ford.com/news-center/view=print/pr-doug-scott2658-marketing-manager-31039").should be_truthy
     end
     it "should allow multiple wildcards" do
       cobweb_links = CobwebLinks.new(:internal_urls => ["http://*.*.domain_one.com/"])
-      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_false
-      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_false
-      cobweb_links.internal?("http://www.marketing.domain_one.com/pageone.html").should be_true
-      cobweb_links.internal?("http://blog.designers.domain_one.com/pagetwo.html").should be_true      
+      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_falsey
+      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_falsey
+      cobweb_links.internal?("http://www.marketing.domain_one.com/pageone.html").should be_truthy
+      cobweb_links.internal?("http://blog.designers.domain_one.com/pagetwo.html").should be_truthy      
     end
     it "should allow multiple country tlds with wildcards" do
       cobweb_links = CobwebLinks.new(:internal_urls => ["http://*.domain_one.*/", "http://*.domain_one.*.*/"])
-      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_true
-      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_true
-      cobweb_links.internal?("http://www.domain_one.co.uk/pageone.html").should be_true
-      cobweb_links.internal?("http://blog.domain_one.co.uk/pageone.html").should be_true
-      cobweb_links.internal?("http://www.domain_one.com.au/pageone.html").should be_true
-      cobweb_links.internal?("http://blog.domain_one.com.au/pageone.html").should be_true
-      cobweb_links.internal?("http://www.domain_one.ie/pageone.html").should be_true
-      cobweb_links.internal?("http://blog.domain_one.ie/pageone.html").should be_true
+      cobweb_links.internal?("http://www.domain_one.com/pageone.html").should be_truthy
+      cobweb_links.internal?("http://blog.domain_one.com/pageone.html").should be_truthy
+      cobweb_links.internal?("http://www.domain_one.co.uk/pageone.html").should be_truthy
+      cobweb_links.internal?("http://blog.domain_one.co.uk/pageone.html").should be_truthy
+      cobweb_links.internal?("http://www.domain_one.com.au/pageone.html").should be_truthy
+      cobweb_links.internal?("http://blog.domain_one.com.au/pageone.html").should be_truthy
+      cobweb_links.internal?("http://www.domain_one.ie/pageone.html").should be_truthy
+      cobweb_links.internal?("http://blog.domain_one.ie/pageone.html").should be_truthy
     end
   end
   

data/spec/cobweb/robots_spec.rb

@@ -19,16 +19,17 @@ describe Robots do
     
     it "should allow urls marked as allow" do
       robot = Robots.new(@options)
-      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_true
+      puts robot.allowed?("http://localhost/globalmarketfinder/asdf.html")
+      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_truthy
     end
     it "should disallow urls specified as disallow" do
       robot = Robots.new(@options)
-      robot.allowed?("http://localhost/globalmarketfinder/").should be_false
-      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_false
+      robot.allowed?("http://localhost/globalmarketfinder/").should be_falsey
+      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_falsey
     end
     it "should allow urls not listed" do
       robot = Robots.new(@options)
-      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_true
+      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_truthy
     end
      
   end
@@ -43,10 +44,10 @@ describe Robots do
     
     it "should disallow all urls" do
       robot = Robots.new(@options)
-      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_false
-      robot.allowed?("http://localhost/globalmarketfinder/").should be_false
-      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_false
-      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_false
+      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_falsey
+      robot.allowed?("http://localhost/globalmarketfinder/").should be_falsey
+      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_falsey
+      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_falsey
     end
     
   end
@@ -61,10 +62,10 @@ describe Robots do
     
     it "should disallow all urls" do
       robot = Robots.new(@options)
-      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_true
-      robot.allowed?("http://localhost/globalmarketfinder/").should be_true
-      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_true
-      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_true
+      robot.allowed?("http://localhost/globalmarketfinder/asdf.html").should be_truthy
+      robot.allowed?("http://localhost/globalmarketfinder/").should be_truthy
+      robot.allowed?("http://localhost/globalmarketfinder/asdf").should be_truthy
+      robot.allowed?("http://localhost/notlistedinrobotsfile").should be_truthy
     end
     
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Stewart McKee
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-02-25 00:00:00.000000000 Z
+date: 2021-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: 1.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.5.2
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -614,7 +614,7 @@ homepage: http://github.com/stewartmckee/cobweb
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -629,9 +629,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.7.7
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites faster than multi-threaded crawlers.  It is also a standalone