spidr 0.1.2 → 0.1.3

data/History.txt

@@ -1,3 +1,9 @@
+=== 0.1.3 / 2009-01-10
+
+* Added the :host options to Spidr::Agent#initialize.
+* Added the Web Spider Obstacle Course files to the Manifest.
+* Aliased Spidr::Agent#visited_urls to Spidr::Agent#history.
+
 === 0.1.2 / 2008-11-06
 
 * Fixed a bug in Page#to_absolute where URLs with no path were not
@@ -19,13 +25,13 @@
 === 0.1.0 / 2008-05-23
 
 * Initial release.
- * Black-list or white-list URLs based upon:
-  * Host name
-  * Port number
-  * Full link
-  * URL extension
- * Provides call-backs for:
-  * Every visited Page.
-  * Every visited URL.
-  * Every visited URL that matches a specified pattern.
+  * Black-list or white-list URLs based upon:
+    * Host name
+    * Port number
+    * Full link
+    * URL extension
+  * Provides call-backs for:
+    * Every visited Page.
+    * Every visited URL.
+    * Every visited URL that matches a specified pattern.
 

data/README.txt

@@ -55,7 +55,7 @@ and easy to use.
 
 The MIT License
 
-Copyright (c) 2008 Hal Brodigan
+Copyright (c) 2008-2009 Hal Brodigan
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/spidr/agent.rb

@@ -33,10 +33,11 @@ module Spidr
     #
     # _options_ may contain the following keys:
     # <tt>:proxy</tt>:: The proxy to use while spidering.
-    # <tt>:user_agent</tt>:: the User-Agent string to send.
+    # <tt>:user_agent</tt>:: The User-Agent string to send.
     # <tt>:referer</tt>:: The referer URL to send.
     # <tt>:delay</tt>:: Duration in seconds to pause between spidering each
     #                   link. Defaults to 0.
+    # <tt>:host</tt>:: The host-name to visit.
     # <tt>:hosts</tt>:: An +Array+ of host patterns to visit.
     # <tt>:ignore_hosts</tt>:: An +Array+ of host patterns to not visit.
     # <tt>:ports</tt>:: An +Array+ of port patterns to visit.
@@ -78,6 +79,10 @@ module Spidr
       @history = []
       @queue = []
 
+      if options[:host]
+        visit_hosts_like(options[:host])
+      end
+
       block.call(self) if block
     end
 
@@ -102,7 +107,7 @@ module Spidr
     # spidering.
     #
     def self.host(name,options={},&block)
-      self.new(options.merge(:hosts => [name.to_s])) do |spider|
+      self.new(options.merge(:host => name)) do |spider|
         block.call(spider) if block
 
         spider.start_at("http://#{name}/")
@@ -118,7 +123,7 @@ module Spidr
     def self.site(url,options={},&block)
       url = URI(url.to_s)
 
-      return self.new(options.merge(:hosts => [url.host])) do |spider|
+      return self.new(options.merge(:host => url.host)) do |spider|
         block.call(spider) if block
 
         spider.start_at(url)
@@ -341,12 +346,7 @@ module Spidr
       return self
     end
 
-    #
-    # Returns the +Array+ of visited URLs.
-    #
-    def visited_urls
-      @history
-    end
+    alias visited_urls history
 
     #
     # Returns the +Array+ of visited URLs.

data/lib/spidr/version.rb

@@ -1,3 +1,3 @@
 module Spidr
-  VERSION = '0.1.2'
+  VERSION = '0.1.3'
 end

data/tasks/spec.rb

@@ -5,3 +5,5 @@ Spec::Rake::SpecTask.new(:spec) do |t|
   t.libs += ['lib', 'spec']
   t.spec_opts = ['--colour', '--format', 'specdoc']
 end
+
+task :default => :spec

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: spidr
 version: !ruby/object:Gem::Version 
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors: 
 - Postmodern
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-11-06 00:00:00 -08:00
+date: 2009-01-10 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -106,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: spidr
-rubygems_version: 1.3.0
+rubygems_version: 1.3.1
 signing_key: 
 specification_version: 2
 summary: Spidr is a versatile Ruby web spidering library that can spider a site, multiple domains, certain links or infinitely