cobweb 0.0.21 → 0.0.22

data/lib/cobweb.rb

@@ -5,8 +5,8 @@ require "addressable/uri"
 require 'digest/sha1'
 require 'base64'
 
-Dir[File.dirname(__FILE__) + '/*.rb'].each do |file|
-  require [File.dirname(__FILE__), File.basename(file, File.extname(file))].join("/")
+Dir[File.dirname(__FILE__) + '/**/*.rb'].each do |file|
+  require file
 end
 
 class Cobweb
@@ -18,6 +18,10 @@ class Cobweb
   # on end of crawl, return statistic hash (could call specified method ?) if single threaded or enqueue to a specified queue the stat hash
   # investigate using event machine for single threaded crawling
   
+  def self.version
+    "0.0.22"
+  end
+  
   def initialize(options = {})
     @options = options
     @options[:follow_redirects] = true unless @options.has_key?(:follow_redirects)
@@ -34,12 +38,12 @@ class Cobweb
   def start(base_url)
     raise ":base_url is required" unless base_url
     request = {
-      :crawl_id => Digest::SHA1.hexdigest(Time.now.to_s),
+      :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
       :url => base_url 
     }  
     
     request.merge!(@options)
-    @redis = NamespacedRedis.new(Redis.new(request[:redis_options]), "cobweb-#{VERSION}-#{request[:crawl_id]}")
+    @redis = NamespacedRedis.new(Redis.new(request[:redis_options]), "cobweb-#{Cobweb.version}-#{request[:crawl_id]}")
     @redis.hset "statistics", "queued_at", DateTime.now
     
     Resque.enqueue(CrawlJob, request)
@@ -53,13 +57,17 @@ class Cobweb
         
     # get the unique id for this request
     unique_id = Digest::SHA1.hexdigest(url.to_s)
-    redirect_limit = options[:redirect_limit]
+    if options.has_key?(:redirect_limit) and !options[:redirect_limit].nil?
+      redirect_limit = options[:redirect_limit].to_i
+    else
+      redirect_limit = 10
+    end
     
     # connect to redis
     if options.has_key? :crawl_id
-      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{VERSION}-#{options[:crawl_id]}")
+      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
     else
-      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{VERSION}")
+      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{Cobweb.version}")
     end
     
     content = {}
@@ -107,7 +115,7 @@ class Cobweb
           raise RedirectError, "Redirect Limit reached" if redirect_limit == 0
 
           # get the content from redirect location
-          content = get(url, redirect_limit)
+          content = get(url, options.merge(:redirect_limit => redirect_limit))
           content[:url] = uri.to_s
           content[:redirect_through] = [] if content[:redirect_through].nil?
           content[:redirect_through].insert(0, url)
@@ -206,9 +214,9 @@ class Cobweb
     
     # connect to redis
     if options.has_key? :crawl_id
-      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{VERSION}-#{options[:crawl_id]}")
+      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
     else
-      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{VERSION}")
+      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{Cobweb.version}")
     end
     
     content = {}

data/lib/crawl_job.rb

@@ -28,8 +28,8 @@ class CrawlJob
   def self.perform(content_request)
     # change all hash keys to symbols    
     content_request.deep_symbolize_keys
-    redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), "cobweb-#{VERSION}-#{content_request[:crawl_id]}")
-    ap redis.namespace
+    redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
+    
     @absolutize = Absolutize.new(content_request[:url], :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
 
     # check we haven't crawled this url before
@@ -95,6 +95,7 @@ class CrawlJob
         redis.sadd "crawled", content_request[:url]
         set_base_url redis, content, content_request[:base_url]
         content[:links].keys.map{|key| content[:links][key]}.flatten.each do |link|
+          link = link.to_s
           unless redis.sismember "crawled", link
             puts "Checking if #{link} matches #{redis.get("base_url")} as internal?" if content_request[:debug]
             if link.to_s.match(Regexp.new("^#{redis.get("base_url")}"))
@@ -129,7 +130,7 @@ class CrawlJob
 
     # detect finished state
 
-    if queue_counter == crawl_counter or content_request[:crawl_limit].to_i <= crawl_counter 
+    if queue_counter == crawl_counter or content_request[:crawl_limit].to_i <= crawl_counter
      
       puts "queue_counter: #{queue_counter}"
       puts "crawl_counter: #{crawl_counter}"

data/lib/stats.rb

@@ -11,7 +11,6 @@ class Stats < Sinatra::Base
     @@status = status
   end
   
-  ap settings.root
   set :views, settings.root + '/../views'
   
   get '/' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.21
+  version: 0.0.22
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70294598137300 !ruby/object:Gem::Requirement
+  requirement: &70101145919340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598137300
+  version_requirements: *70101145919340
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70294598136460 !ruby/object:Gem::Requirement
+  requirement: &70101145918920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598136460
+  version_requirements: *70101145918920
 - !ruby/object:Gem::Dependency
   name: absolutize
-  requirement: &70294598135800 !ruby/object:Gem::Requirement
+  requirement: &70101145918500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598135800
+  version_requirements: *70101145918500
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70294598135140 !ruby/object:Gem::Requirement
+  requirement: &70101145934440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598135140
+  version_requirements: *70101145934440
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70294598134440 !ruby/object:Gem::Requirement
+  requirement: &70101145934020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598134440
+  version_requirements: *70101145934020
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70294598133760 !ruby/object:Gem::Requirement
+  requirement: &70101145933580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598133760
+  version_requirements: *70101145933580
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70294598132820 !ruby/object:Gem::Requirement
+  requirement: &70101145933160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598132820
+  version_requirements: *70101145933160
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70294598131980 !ruby/object:Gem::Requirement
+  requirement: &70101145932740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598131980
+  version_requirements: *70101145932740
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70294598131160 !ruby/object:Gem::Requirement
+  requirement: &70101145932320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598131160
+  version_requirements: *70101145932320
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70294598130540 !ruby/object:Gem::Requirement
+  requirement: &70101145931900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70294598130540
+  version_requirements: *70101145931900
 description: 
 email: stewart@rockwellcottage.com
 executables: []