cobweb 0.0.17 → 0.0.18

data/README.textile

@@ -1,14 +1,24 @@
 
-h1. Cobweb v0.0.13
+h1. Cobweb v0.0.18
 
 h2. Intro 
   
   CobWeb has three methods of running.  Firstly it is a http client that allows get and head requests returning a hash of data relating to the requested resource.  The second main function is to utilize this combined with the power of Resque to cluster the crawls allowing you crawl quickly.  Lastly you can run the crawler with a block that uses each of the pages found in the crawl.
   
+h3. Resque
+
   When running on resque, passing in a Class and queue name it will enqueue all resources to this queue for processing, passing in the hash it has generated.  You then implement the perform method to process the resource for your own application.
   
-  Documentation for running with a block will come soon..  Check out the CobwebCrawler spec for hints.
+h3. Standalone
   
+  CobwebCrawler takes the same options as cobweb itself, so you can use any of the options available for that.  An example is listed below.
+
+bq. crawler = CobwebCrawler.new(:cache => 600);
+bq. stats = crawler.crawl("http://www.pepsico.com")
+
+  While the crawler is running, you can view statistics on http://localhost:4567
+
+h3. Data Returned
   The data available in the returned hash are:  
   
   * :url - url of the resource requested

data/lib/cobweb.rb

@@ -1,5 +1,4 @@
 require 'rubygems'
-require 'bundler/setup'
 require 'uri'
 require 'resque'
 require "addressable/uri"
@@ -75,7 +74,7 @@ class Cobweb
       
       # retrieve data
       unless @http && @http.address == uri.host && @http.port == uri.inferred_port
-        puts "Creating connection to #{uri.host}..."
+        puts "Creating connection to #{uri.host}..." unless @options[:quiet]
         @http = Net::HTTP.new(uri.host, uri.inferred_port)
       end
       if uri.scheme == "https"

data/lib/stats.rb

@@ -1,4 +1,3 @@
-require 'bundler/setup'
 require 'sinatra'
 require 'haml'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.18
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70173021660540 !ruby/object:Gem::Requirement
+  requirement: &70110522428200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021660540
+  version_requirements: *70110522428200
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70173021659920 !ruby/object:Gem::Requirement
+  requirement: &70110522427560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021659920
+  version_requirements: *70110522427560
 - !ruby/object:Gem::Dependency
   name: absolutize
-  requirement: &70173021659420 !ruby/object:Gem::Requirement
+  requirement: &70110522427020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021659420
+  version_requirements: *70110522427020
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70173021658760 !ruby/object:Gem::Requirement
+  requirement: &70110522426360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021658760
+  version_requirements: *70110522426360
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70173021658080 !ruby/object:Gem::Requirement
+  requirement: &70110522425700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021658080
+  version_requirements: *70110522425700
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70173021657320 !ruby/object:Gem::Requirement
+  requirement: &70110522424920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021657320
+  version_requirements: *70110522424920
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70173021654680 !ruby/object:Gem::Requirement
+  requirement: &70110522424300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021654680
+  version_requirements: *70110522424300
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70173021654060 !ruby/object:Gem::Requirement
+  requirement: &70110522421640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021654060
+  version_requirements: *70110522421640
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70173021653560 !ruby/object:Gem::Requirement
+  requirement: &70110522421140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70173021653560
+  version_requirements: *70110522421140
 description: 
 email: stewart@rockwellcottage.com
 executables: []