cobweb 1.0.11 → 1.0.12

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 1.0.11
+  version: 1.0.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-18 00:00:00.000000000 Z
+date: 2013-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: resque
-  requirement: &70274619912400 !ruby/object:Gem::Requirement
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,21 +21,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619912400
-- !ruby/object:Gem::Dependency
-  name: redis
-  requirement: &70274619906680 !ruby/object:Gem::Requirement
+  version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *70274619906680
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70274619897540 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +37,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619897540
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70274619888180 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +53,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619888180
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70274619880820 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +69,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619880820
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70274619877520 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +85,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619877520
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70274619876520 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +101,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619876520
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70274619875960 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +117,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619875960
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70274619875340 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +133,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619875340
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70274619874800 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,10 +149,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619874800
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70274619874180 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -131,10 +165,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619874180
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: slop
-  requirement: &70274619873340 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -142,7 +181,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70274619873340
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more performant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface
@@ -156,16 +200,17 @@ files:
 - spec/cobweb/cobweb_crawl_helper_spec.rb
 - spec/cobweb/cobweb_crawl_spec.rb
 - spec/cobweb/cobweb_crawler_spec.rb
-- spec/cobweb/cobweb_job_spec.rb
 - spec/cobweb/cobweb_links_spec.rb
 - spec/cobweb/cobweb_spec.rb
 - spec/cobweb/content_link_parser_spec.rb
+- spec/cobweb/crawl_job_spec.rb
+- spec/cobweb/crawl_worker_spec.rb
 - spec/cobweb/robots_spec.rb
-- spec/cobweb/site_test_spec.rb.tmp
+- spec/http_stubs.rb
 - spec/samples/robots.txt
 - spec/samples/sample_html_links.html
 - spec/samples/sample_server.rb
-- spec/samples/sample_site/boxgrid.html
+- spec/samples/sample_site/boxgrid>withsillyname.html
 - spec/samples/sample_site/css/accordion.css
 - spec/samples/sample_site/css/datatable.css
 - spec/samples/sample_site/css/datepicker.css
@@ -343,8 +388,12 @@ files:
 - lib/cobweb_version.rb
 - lib/content_link_parser.rb
 - lib/crawl.rb
+- lib/crawl_finished_worker.rb
+- lib/crawl_helper.rb
 - lib/crawl_job.rb
 - lib/crawl_object.rb
+- lib/crawl_process_worker.rb
+- lib/crawl_worker.rb
 - lib/document.rb
 - lib/encoding_safe_process_job.rb
 - lib/export_command.rb
@@ -353,6 +402,7 @@ files:
 - lib/report_command.rb
 - lib/robots.rb
 - lib/server.rb
+- lib/sidekiq/cobweb_helper.rb
 - lib/stats.rb
 - lib/string.rb
 - lib/uri_helper.rb
@@ -530,7 +580,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Cobweb is a web crawler that can use resque to cluster crawls to quickly

data/spec/cobweb/site_test_spec.rb.tmp

@@ -1,101 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
-
-describe Cobweb, :local_only => true do
-
-  before(:all) do
-    #store all existing resque process ids so we don't kill them afterwards
-    @existing_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
-
-    # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES
-    puts "Starting Workers... Please Wait..."
-    `mkdir log`
-    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=3 QUEUE=cobweb_crawl_job > log/output.log &")
-    puts "Workers Started."
-
-  end
-
-  before(:each) do
-    @base_url = "http://localhost:3532/"
-    @base_page_count = 77
-    clear_queues
-  end
-
-  describe "with a crawl limit" do
-    before(:each) do
-      @request = {
-        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
-        :quiet => true,
-        :cache => nil,
-        :use_encoding_safe_process_job => true,
-        :crawl_limit_by_page => true
-      }
-    end
-    
-    describe "on ancestry.com.au" do
-      describe "limited to 100" do
-        before(:each) do
-          @request[:crawl_limit] = 100
-          @request[:valid_mime_types] = ["text/html"]
-          @cobweb = Cobweb.new @request        
-        end
-        
-        it "should crawl 100 pages" do
-          crawl = @cobweb.start("http://www.ancestry.com.au/")
-          @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
-          wait_for_crawl_finished crawl[:crawl_id], 180
-          puts "Crawled #{Resque.size("encoding_safe_process_job")} pages."
-        end
-      end
-      
-      describe "limited to 999" do
-        before(:each) do
-          @request[:crawl_limit] = 999
-          @cobweb = Cobweb.new @request        
-        end
-      
-        it "should crawl 999 pages" do
-          crawl = @cobweb.start("http://www.ancestry.com.au/")
-          @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
-          wait_for_crawl_finished crawl[:crawl_id], 720
-          puts "Crawled #{Resque.size("encoding_safe_process_job")} pages."
-        end
-      end
-    __END__
-    
-  end
-
-  after(:all) do
-
-    @all_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
-    command = "kill -9 #{(@all_processes - @existing_processes).join(" ")}"
-    IO.popen(command)
-
-    clear_queues
-  end
-
-end
-
-def wait_for_crawl_finished(crawl_id, timeout=20)
-  counter = 0
-  start_time = Time.now
-  while(running?(crawl_id) && Time.now < start_time + timeout) do
-      sleep 0.5
-    end
-    if Time.now > start_time + timeout
-      raise "End of crawl not detected"
-    end
-  end
-
-  def running?(crawl_id)
-    @stat.get_status != CobwebCrawlHelper::FINISHED and @stat.get_status != CobwebCrawlHelper::CANCELLED
-  end
-
-  def clear_queues
-    Resque.queues.each do |queue|
-      Resque.remove_queue(queue)
-    end
-
-    Resque.size("cobweb_process_job").should == 0
-    Resque.size("cobweb_finished_job").should == 0
-    Resque.peek("cobweb_process_job", 0, 200).should be_empty
-  end