spidr 0.2.0 → 0.2.1

data/History.txt

@@ -1,3 +1,34 @@
+=== 0.2.1 / 2009-11-25
+
+* Added Spidr::Events#every_ok_page.
+* Added Spidr::Events#every_redirect_page.
+* Added Spidr::Events#every_timedout_page.
+* Added Spidr::Events#every_bad_request_page.
+* Added Spidr::Events#every_unauthorized_page.
+* Added Spidr::Events#every_forbidden_page.
+* Added Spidr::Events#every_missing_page.
+* Added Spidr::Events#every_internal_server_error_page.
+* Added Spidr::Events#every_txt_page.
+* Added Spidr::Events#every_html_page.
+* Added Spidr::Events#every_xml_page.
+* Added Spidr::Events#every_xsl_page.
+* Added Spidr::Events#every_doc.
+* Added Spidr::Events#every_html_doc.
+* Added Spidr::Events#every_xml_doc.
+* Added Spidr::Events#every_xsl_doc.
+* Added Spidr::Events#every_rss_doc.
+* Added Spidr::Events#every_atom_doc.
+* Added Spidr::Events#every_javascript_page.
+* Added Spidr::Events#every_css_page.
+* Added Spidr::Events#every_rss_page.
+* Added Spidr::Events#every_atom_page.
+* Added Spidr::Events#every_ms_word_page.
+* Added Spidr::Events#every_pdf_page.
+* Added Spidr::Events#every_zip_page.
+* Fixed a bug where Spidr::Agent#delay was not being used to delay
+  requesting pages.
+* Spider +link+ and +script+ tags in HTML pages (thanks Nick Plante).
+
 === 0.2.0 / 2009-10-10
 
 * Added URI.expand_path.

data/README.txt

@@ -1,8 +1,10 @@
 = Spidr
 
-* http://spidr.rubyforge.org/
-* http://github.com/postmodern/spidr/
-* Postmodern (postmodern.mod3 at gmail.com)
+* http://spidr.rubyforge.org
+* http://github.com/postmodern/spidr
+* http://github.com/postmodern/spidr/issues
+* http://groups.google.com/group/spidr
+* irc.freenode.net #spidr
 
 == DESCRIPTION:
 
@@ -56,7 +58,7 @@ and easy to use.
       'http://company.com/',
       :hosts => [
         'company.com',
-	/host\d\.company\.com/
+        /host\d\.company\.com/
       ]
     )
 
@@ -90,10 +92,10 @@ and easy to use.
         puts "[-] #{page.url}"
 
         page.search('//meta').each do |meta|
-	  name = (meta.attributes['name'] || meta.attributes['http-equiv'])
-	  value = meta.attributes['content']
+          name = (meta.attributes['name'] || meta.attributes['http-equiv'])
+          value = meta.attributes['content']
 
-	  puts "    #{name} = #{value}"
+          puts "    #{name} = #{value}"
 	end
       end
     end
@@ -101,8 +103,8 @@ and easy to use.
 * Print out the titles from every page:
 
     Spidr.site('http://www.rubypulse.com/') do |spider|
-      spider.every_page do |page|
-        puts page.title if page.html?
+      spider.every_html_page do |page|
+        puts page.title
       end
     end
 
@@ -119,16 +121,16 @@ and easy to use.
 * Pause the spider on a forbidden page:
 
     spider = Spidr.host('overnight.startup.com') do |spider|
-      spider.every_page do |page|
-        spider.pause! if page.forbidden?
+      spider.every_forbidden_page do |page|
+        spider.pause!
       end
     end
 
 * Skip the processing of a page:
 
     Spidr.host('sketchy.content.com') do |spider|
-      spider.every_page do |page|
-        spider.skip_page! if page.not_found?
+      spider.every_missing_page do |page|
+        spider.skip_page!
       end
     end
 
@@ -137,8 +139,8 @@ and easy to use.
     Spidr.host('sketchy.content.com') do |spider|
       spider.every_url do |url|
         if url.path.split('/').find { |dir| dir.to_i > 1000 }
-	  spider.skip_link!
-	end
+          spider.skip_link!
+        end
       end
     end
 

data/Rakefile

@@ -18,7 +18,7 @@ Hoe.spec('spidr') do
 
   self.extra_dev_deps = [
     ['rspec', '>=1.2.8'],
-    ['yard', '>=0.2.3.5']
+    ['yard', '>=0.4.0']
   ]
 
   self.spec_extras = {:has_rdoc => 'yard'}

data/lib/spidr/agent.rb

@@ -458,6 +458,8 @@ module Spidr
       path += "?#{url.query}" if url.query
 
       begin
+        sleep(@delay) if @delay > 0
+
         get_session(url.scheme,host,port) do |sess|
           headers = {}
           headers['User-Agent'] = @user_agent if @user_agent

data/lib/spidr/events.rb

@@ -56,6 +56,20 @@ module Spidr
       return self
     end
 
+    #
+    # Pass the headers from every response the agent receives to a given
+    # block.
+    #
+    # @yield [headers]
+    #   The block will be passed the headers of every response.
+    #
+    # @yieldparam [Hash] headers
+    #   The headers from a response.
+    #
+    def all_headers(&block)
+      every_page { |page| block.call(page.headers) }
+    end
+
     #
     # Pass every page that the agent visits to a given block.
     #
@@ -71,17 +85,419 @@ module Spidr
     end
 
     #
-    # Pass the headers from every response the agent receives to a given
+    # Pass every OK page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every OK page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_ok_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.ok?)
+      end
+    end
+
+    #
+    # Pass every Redirect page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every Redirect page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_redirect_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.redirect?)
+      end
+    end
+
+    #
+    # Pass every Timeout page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every Timeout page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_timedout_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.timedout?)
+      end
+    end
+
+    #
+    # Pass every Bad Request page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every Bad Request page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_bad_request_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.bad_request?)
+      end
+    end
+
+    #
+    # Pass every Unauthorized page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every Unauthorized page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_unauthorized_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.unauthorized?)
+      end
+    end
+
+    #
+    # Pass every Forbidden page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every Forbidden page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_forbidden_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.forbidden?)
+      end
+    end
+
+    #
+    # Pass every Missing page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every Missing page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_missing_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.missing?)
+      end
+    end
+
+    #
+    # Pass every Internal Server Error page that the agent visits to a
+    # given block.
+    #
+    # @yield [page]
+    #   The block will be passed every Internal Server Error page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_internal_server_error_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.had_internal_server_error?)
+      end
+    end
+
+    #
+    # Pass every Plain Text page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every Plain Text page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_txt_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.txt?)
+      end
+    end
+
+    #
+    # Pass every HTML page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every HTML page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_html_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.html?)
+      end
+    end
+
+    #
+    # Pass every XML page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every XML page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_xml_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.xml?)
+      end
+    end
+
+    #
+    # Pass every XML Stylesheet (XSL) page that the agent visits to a
+    # given block.
+    #
+    # @yield [page]
+    #   The block will be passed every XML Stylesheet (XSL) page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_xsl_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.xsl?)
+      end
+    end
+
+    #
+    # Pass every HTML or XML document that the agent parses to a given
     # block.
     #
-    # @yield [headers]
-    #   The block will be passed the headers of every response.
+    # @yield [doc]
+    #   The block will be passed every HTML or XML document parsed.
     #
-    # @yieldparam [Hash] headers
-    #   The headers from a response.
+    # @yieldparam [Nokogiri::HTML::Document, Nokogiri::XML::Document] doc
+    #   A parsed HTML or XML document.
     #
-    def all_headers(&block)
-      every_page { |page| block.call(page.headers) }
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html
+    #
+    def every_doc(&block)
+      every_page do |page|
+        if block
+          if (doc = page.doc)
+            block.call(doc)
+          end
+        end
+      end
+    end
+
+    #
+    # Pass every HTML document that the agent parses to a given block.
+    #
+    # @yield [doc]
+    #   The block will be passed every HTML document parsed.
+    #
+    # @yieldparam [Nokogiri::HTML::Document] doc
+    #   A parsed HTML document.
+    #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html
+    #
+    def every_html_doc(&block)
+      every_page do |page|
+        if (block && page.html?)
+          if (doc = page.doc)
+            block.call(doc)
+          end
+        end
+      end
+    end
+
+    #
+    # Pass every XML document that the agent parses to a given block.
+    #
+    # @yield [doc]
+    #   The block will be passed every XML document parsed.
+    #
+    # @yieldparam [Nokogiri::XML::Document] doc
+    #   A parsed XML document.
+    #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
+    #
+    def every_xml_doc(&block)
+      every_page do |page|
+        if (block && page.xml?)
+          if (doc = page.doc)
+            block.call(doc)
+          end
+        end
+      end
+    end
+
+    #
+    # Pass every XML Stylesheet (XSL) that the agent parses to a given
+    # block.
+    #
+    # @yield [doc]
+    #   The block will be passed every XSL Stylesheet (XSL) parsed.
+    #
+    # @yieldparam [Nokogiri::XML::Document] doc
+    #   A parsed XML document.
+    #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
+    #
+    def every_xsl_doc(&block)
+      every_page do |page|
+        if (block && page.xsl?)
+          if (doc = page.doc)
+            block.call(doc)
+          end
+        end
+      end
+    end
+
+    #
+    # Pass every RSS document that the agent parses to a given block.
+    #
+    # @yield [doc]
+    #   The block will be passed every RSS document parsed.
+    #
+    # @yieldparam [Nokogiri::XML::Document] doc
+    #   A parsed XML document.
+    #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
+    #
+    def every_rss_doc(&block)
+      every_page do |page|
+        if (block && page.rss?)
+          if (doc = page.doc)
+            block.call(doc)
+          end
+        end
+      end
+    end
+
+    #
+    # Pass every Atom document that the agent parses to a given block.
+    #
+    # @yield [doc]
+    #   The block will be passed every Atom document parsed.
+    #
+    # @yieldparam [Nokogiri::XML::Document] doc
+    #   A parsed XML document.
+    #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
+    #
+    def every_atom_doc(&block)
+      every_page do |page|
+        if (block && page.atom?)
+          if (doc = page.doc)
+            block.call(doc)
+          end
+        end
+      end
+    end
+
+    #
+    # Pass every JavaScript page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every JavaScript page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_javascript_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.javascript?)
+      end
+    end
+
+    #
+    # Pass every CSS page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every CSS page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_css_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.css?)
+      end
+    end
+
+    #
+    # Pass every RSS feed that the agent visits to a given block.
+    #
+    # @yield [feed]
+    #   The block will be passed every RSS feed visited.
+    #
+    # @yieldparam [Page] feed
+    #   A visited page.
+    #
+    def every_rss_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.rss?)
+      end
+    end
+
+    #
+    # Pass every Atom feed that the agent visits to a given block.
+    #
+    # @yield [feed]
+    #   The block will be passed every Atom feed visited.
+    #
+    # @yieldparam [Page] feed
+    #   A visited page.
+    #
+    def every_atom_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.atom?)
+      end
+    end
+
+    #
+    # Pass every MS Word page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every MS Word page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_ms_word_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.ms_word?)
+      end
+    end
+
+    #
+    # Pass every PDF page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every PDF page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_pdf_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.pdf?)
+      end
+    end
+
+    #
+    # Pass every ZIP page that the agent visits to a given block.
+    #
+    # @yield [page]
+    #   The block will be passed every ZIP page visited.
+    #
+    # @yieldparam [Page] page
+    #   A visited page.
+    #
+    def every_zip_page(&block)
+      every_page do |page|
+        block.call(page) if (block && page.zip?)
+      end
     end
   end
 end

data/lib/spidr/page.rb

@@ -173,6 +173,16 @@ module Spidr
       (content_type =~ /text\/xml/) == 0
     end
 
+    #
+    # Determines if the page is XML Stylesheet (XSL).
+    #
+    # @return [Boolean]
+    #   Specifies whether the page is XML Stylesheet (XSL).
+    #
+    def xsl?
+      (content_type =~ /text\/xsl/) == 0
+    end
+
     #
     # Determines if the page is JavaScript.
     #
@@ -261,13 +271,16 @@ module Spidr
     #   Returns +nil+ if the page is neither HTML, XML, RSS, Atom or if
     #   the page could not be parsed properly.
     #
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
+    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html
+    #
     def doc
       return nil if (body.nil? || body.empty?)
 
       begin
         if html?
           return @doc ||= Nokogiri::HTML(body)
-        elsif (xml? || rss? || atom?)
+        elsif (xml? || xsl? || rss? || atom?)
           return @doc ||= Nokogiri::XML(body)
         end
       rescue
@@ -373,6 +386,14 @@ module Spidr
         doc.search('iframe[@src]').each do |iframe|
           add_url.call(iframe.get_attribute('src'))
         end
+
+        doc.search('link[@href]').each do |link|
+          add_url.call(link.get_attribute('href'))
+        end
+
+        doc.search('script[@src]').each do |script|
+          add_url.call(script.get_attribute('src'))
+        end
       end
 
       return urls

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.2.0'
+  VERSION = '0.2.1'
 end

data/tasks/yard.rb

@@ -9,4 +9,4 @@ YARD::Rake::YardocTask.new do |t|
   ]
 end
 
-task :docs => :yardoc
+task :docs => :yard

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: spidr
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors: 
 - Postmodern
@@ -30,7 +30,7 @@ cert_chain:
   pDj+ws7QjtH/Qcrr1l9jfN0ehDs=
   -----END CERTIFICATE-----
 
-date: 2009-10-10 00:00:00 -07:00
+date: 2009-11-25 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -61,7 +61,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.2.3.5
+        version: 0.4.0
     version: 
 - !ruby/object:Gem::Dependency 
   name: hoe
@@ -154,7 +154,7 @@ files:
 - static/course/frames/frame_next.html
 - static/course/specs.json
 has_rdoc: yard
-homepage: http://spidr.rubyforge.org/
+homepage: http://spidr.rubyforge.org
 licenses: []
 
 post_install_message: