spidr_epg 1.0.0

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2008-2011 Hal Brodigan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,193 @@
+# 在这个gems基础上对它的源码进行了部分的修改已符合自己的需求
+
+
+# Spidr
+
+* [Homepage](http://spidr.rubyforge.org/)
+* [Source](http://github.com/postmodern/spidr)
+* [Issues](http://github.com/postmodern/spidr/issues)
+* [Mailing List](http://groups.google.com/group/spidr)
+* [IRC](http://webchat.freenode.net/?channels=spidr&uio=d4)
+
+## Description
+
+Spidr is a versatile Ruby web spidering library that can spider a site,
+multiple domains, certain links or infinitely. Spidr is designed to be fast
+and easy to use.
+
+## Features
+
+* Follows:
+  * a tags.
+  * iframe tags.
+  * frame tags.
+  * Cookie protected links.
+  * HTTP 300, 301, 302, 303 and 307 Redirects.
+  * Meta-Refresh Redirects.
+  * HTTP Basic Auth protected links.
+* Black-list or white-list URLs based upon:
+  * URL scheme.
+  * Host name
+  * Port number
+  * Full link
+  * URL extension
+* Provides call-backs for:
+  * Every visited Page.
+  * Every visited URL.
+  * Every visited URL that matches a specified pattern.
+  * Every origin and destination URI of a link.
+  * Every URL that failed to be visited.
+* Provides action methods to:
+  * Pause spidering.
+  * Skip processing of pages.
+  * Skip processing of links.
+* Restore the spidering queue and history from a previous session.
+* Custom User-Agent strings.
+* Custom proxy settings.
+* HTTPS support.
+
+## Examples
+
+Start spidering from a URL:
+
+    Spidr.start_at('http://tenderlovemaking.com/')
+
+Spider a host:
+
+    Spidr.host('coderrr.wordpress.com')
+
+Spider a site:
+
+    Spidr.site('http://rubyflow.com/')
+
+Spider multiple hosts:
+
+    Spidr.start_at(
+      'http://company.com/',
+      :hosts => [
+        'company.com',
+        /host\d\.company\.com/
+      ]
+    )
+
+Do not spider certain links:
+
+    Spidr.site('http://matasano.com/', :ignore_links => [/log/])
+
+Do not spider links on certain ports:
+
+    Spidr.site(
+      'http://sketchy.content.com/',
+      :ignore_ports => [8000, 8010, 8080]
+    )
+
+Print out visited URLs:
+
+    Spidr.site('http://rubyinside.org/') do |spider|
+      spider.every_url { |url| puts url }
+    end
+
+Build a URL map of a site:
+
+    url_map = Hash.new { |hash,key| hash[key] = [] }
+
+    Spidr.site('http://intranet.com/') do |spider|
+      spider.every_link do |origin,dest|
+        url_map[dest] << origin
+      end
+    end
+
+Print out the URLs that could not be requested:
+
+    Spidr.site('http://sketchy.content.com/') do |spider|
+      spider.every_failed_url { |url| puts url }
+    end
+
+Finds all pages which have broken links:
+
+    url_map = Hash.new { |hash,key| hash[key] = [] }
+
+    spider = Spidr.site('http://intranet.com/') do |spider|
+      spider.every_link do |origin,dest|
+        url_map[dest] << origin
+      end
+    end
+
+    spider.failures.each do |url|
+      puts "Broken link #{url} found in:"
+
+      url_map[url].each { |page| puts "  #{page}" }
+    end
+
+Search HTML and XML pages:
+
+    Spidr.site('http://company.withablog.com/') do |spider|
+      spider.every_page do |page|
+        puts "[-] #{page.url}"
+
+        page.search('//meta').each do |meta|
+          name = (meta.attributes['name'] || meta.attributes['http-equiv'])
+          value = meta.attributes['content']
+
+          puts "    #{name} = #{value}"
+        end
+      end
+    end
+
+Print out the titles from every page:
+
+    Spidr.site('http://www.rubypulse.com/') do |spider|
+      spider.every_html_page do |page|
+        puts page.title
+      end
+    end
+
+Find what kinds of web servers a host is using, by accessing the headers:
+
+    servers = Set[]
+
+    Spidr.host('generic.company.com') do |spider|
+      spider.all_headers do |headers|
+        servers << headers['server']
+      end
+    end
+
+Pause the spider on a forbidden page:
+
+    spider = Spidr.host('overnight.startup.com') do |spider|
+      spider.every_forbidden_page do |page|
+        spider.pause!
+      end
+    end
+
+Skip the processing of a page:
+
+    Spidr.host('sketchy.content.com') do |spider|
+      spider.every_missing_page do |page|
+        spider.skip_page!
+      end
+    end
+
+Skip the processing of links:
+
+    Spidr.host('sketchy.content.com') do |spider|
+      spider.every_url do |url|
+        if url.path.split('/').find { |dir| dir.to_i > 1000 }
+          spider.skip_link!
+        end
+      end
+    end
+
+## Requirements
+
+* [nokogiri](http://nokogiri.rubyforge.org/) ~> 1.3
+
+## Install
+
+    $ sudo gem install spidr
+
+## License
+
+Copyright (c) 2008-2011 Hal Brodigan
+
+See {file:LICENSE.txt} for license information.

data/README.md~

@@ -0,0 +1,190 @@
+# Spidr
+
+* [Homepage](http://spidr.rubyforge.org/)
+* [Source](http://github.com/postmodern/spidr)
+* [Issues](http://github.com/postmodern/spidr/issues)
+* [Mailing List](http://groups.google.com/group/spidr)
+* [IRC](http://webchat.freenode.net/?channels=spidr&uio=d4)
+
+## Description
+
+Spidr is a versatile Ruby web spidering library that can spider a site,
+multiple domains, certain links or infinitely. Spidr is designed to be fast
+and easy to use.
+
+## Features
+
+* Follows:
+  * a tags.
+  * iframe tags.
+  * frame tags.
+  * Cookie protected links.
+  * HTTP 300, 301, 302, 303 and 307 Redirects.
+  * Meta-Refresh Redirects.
+  * HTTP Basic Auth protected links.
+* Black-list or white-list URLs based upon:
+  * URL scheme.
+  * Host name
+  * Port number
+  * Full link
+  * URL extension
+* Provides call-backs for:
+  * Every visited Page.
+  * Every visited URL.
+  * Every visited URL that matches a specified pattern.
+  * Every origin and destination URI of a link.
+  * Every URL that failed to be visited.
+* Provides action methods to:
+  * Pause spidering.
+  * Skip processing of pages.
+  * Skip processing of links.
+* Restore the spidering queue and history from a previous session.
+* Custom User-Agent strings.
+* Custom proxy settings.
+* HTTPS support.
+
+## Examples
+
+Start spidering from a URL:
+
+    Spidr.start_at('http://tenderlovemaking.com/')
+
+Spider a host:
+
+    Spidr.host('coderrr.wordpress.com')
+
+Spider a site:
+
+    Spidr.site('http://rubyflow.com/')
+
+Spider multiple hosts:
+
+    Spidr.start_at(
+      'http://company.com/',
+      :hosts => [
+        'company.com',
+        /host\d\.company\.com/
+      ]
+    )
+
+Do not spider certain links:
+
+    Spidr.site('http://matasano.com/', :ignore_links => [/log/])
+
+Do not spider links on certain ports:
+
+    Spidr.site(
+      'http://sketchy.content.com/',
+      :ignore_ports => [8000, 8010, 8080]
+    )
+
+Print out visited URLs:
+
+    Spidr.site('http://rubyinside.org/') do |spider|
+      spider.every_url { |url| puts url }
+    end
+
+Build a URL map of a site:
+
+    url_map = Hash.new { |hash,key| hash[key] = [] }
+
+    Spidr.site('http://intranet.com/') do |spider|
+      spider.every_link do |origin,dest|
+        url_map[dest] << origin
+      end
+    end
+
+Print out the URLs that could not be requested:
+
+    Spidr.site('http://sketchy.content.com/') do |spider|
+      spider.every_failed_url { |url| puts url }
+    end
+
+Finds all pages which have broken links:
+
+    url_map = Hash.new { |hash,key| hash[key] = [] }
+
+    spider = Spidr.site('http://intranet.com/') do |spider|
+      spider.every_link do |origin,dest|
+        url_map[dest] << origin
+      end
+    end
+
+    spider.failures.each do |url|
+      puts "Broken link #{url} found in:"
+
+      url_map[url].each { |page| puts "  #{page}" }
+    end
+
+Search HTML and XML pages:
+
+    Spidr.site('http://company.withablog.com/') do |spider|
+      spider.every_page do |page|
+        puts "[-] #{page.url}"
+
+        page.search('//meta').each do |meta|
+          name = (meta.attributes['name'] || meta.attributes['http-equiv'])
+          value = meta.attributes['content']
+
+          puts "    #{name} = #{value}"
+        end
+      end
+    end
+
+Print out the titles from every page:
+
+    Spidr.site('http://www.rubypulse.com/') do |spider|
+      spider.every_html_page do |page|
+        puts page.title
+      end
+    end
+
+Find what kinds of web servers a host is using, by accessing the headers:
+
+    servers = Set[]
+
+    Spidr.host('generic.company.com') do |spider|
+      spider.all_headers do |headers|
+        servers << headers['server']
+      end
+    end
+
+Pause the spider on a forbidden page:
+
+    spider = Spidr.host('overnight.startup.com') do |spider|
+      spider.every_forbidden_page do |page|
+        spider.pause!
+      end
+    end
+
+Skip the processing of a page:
+
+    Spidr.host('sketchy.content.com') do |spider|
+      spider.every_missing_page do |page|
+        spider.skip_page!
+      end
+    end
+
+Skip the processing of links:
+
+    Spidr.host('sketchy.content.com') do |spider|
+      spider.every_url do |url|
+        if url.path.split('/').find { |dir| dir.to_i > 1000 }
+          spider.skip_link!
+        end
+      end
+    end
+
+## Requirements
+
+* [nokogiri](http://nokogiri.rubyforge.org/) ~> 1.3
+
+## Install
+
+    $ sudo gem install spidr
+
+## License
+
+Copyright (c) 2008-2011 Hal Brodigan
+
+See {file:LICENSE.txt} for license information.

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rubygems'
+
+begin
+  require 'bundler'
+rescue LoadError => e
+  STDERR.puts e.message
+  STDERR.puts "Run `gem install bundler` to install Bundler."
+  exit e.status_code
+end
+
+begin
+  Bundler.setup(:development)
+rescue Bundler::BundlerError => e
+  STDERR.puts e.message
+  STDERR.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'rake'
+
+require 'rubygems/tasks'
+Gem::Tasks.new
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/gemspec.yml

@@ -0,0 +1,19 @@
+name: spidr
+summary: A versatile Ruby web spidering library
+description:
+  Spidr is a versatile Ruby web spidering library that can spider a site,
+  multiple domains, certain links or infinitely. Spidr is designed to be
+  fast and easy to use.
+
+license: MIT
+authors: Postmodern
+email: postmodern.mod3@gmail.com
+homepage: http://github.com/postmodern/spidr
+has_yard: true
+
+dependencies:
+  nokogiri: ~> 1.3
+
+development_dependencies:
+  bundler: ~> 1.0
+  yard: ~> 0.7

data/lib/spidr/actions/actions.rb

@@ -0,0 +1,83 @@
+require 'spidr/actions/exceptions/paused'
+require 'spidr/actions/exceptions/skip_link'
+require 'spidr/actions/exceptions/skip_page'
+
+module Spidr
+  #
+  # The {Actions} module adds methods to {Agent} for controlling the
+  # spidering of links.
+  #
+  module Actions
+    #
+    # Continue spidering.
+    #
+    # @yield [page]
+    #   If a block is given, it will be passed every page visited.
+    #
+    # @yieldparam [Page] page
+    #   The page to be visited.
+    #
+    def continue!(&block)
+      @paused = false
+      return run(&block)
+    end
+
+    #
+    # Sets the pause state of the agent.
+    #
+    # @param [Boolean] state
+    #   The new pause state of the agent.
+    #
+    def pause=(state)
+      @paused = state
+    end
+
+    #
+    # Pauses the agent, causing spidering to temporarily stop.
+    #
+    # @raise [Paused]
+    #   Indicates to the agent, that it should pause spidering.
+    #
+    def pause!
+      @paused = true
+      raise(Paused)
+    end
+
+    #
+    # Determines whether the agent is paused.
+    #
+    # @return [Boolean]
+    #   Specifies whether the agent is paused.
+    #
+    def paused?
+      @paused == true
+    end
+
+    #
+    # Causes the agent to skip the link being enqueued.
+    #
+    # @raise [SkipLink]
+    #   Indicates to the agent, that the current link should be skipped,
+    #   and not enqueued or visited.
+    #
+    def skip_link!
+      raise(SkipLink)
+    end
+
+    #
+    # Causes the agent to skip the page being visited.
+    #
+    # @raise [SkipPage]
+    #   Indicates to the agent, that the current page should be skipped.
+    #
+    def skip_page!
+      raise(SkipPage)
+    end
+
+    protected
+
+    def initialize_actions(options={})
+      @paused = false
+    end
+  end
+end

data/lib/spidr/actions/exceptions/action.rb

@@ -0,0 +1,9 @@
+module Spidr
+  module Actions
+    #
+    # The base {Actions} exception class.
+    #
+    class Action < RuntimeError
+    end
+  end
+end

data/lib/spidr/actions/exceptions/paused.rb

@@ -0,0 +1,11 @@
+require 'spidr/actions/exceptions/action'
+
+module Spidr
+  module Actions
+    #
+    # An {Actions} exception class used to pause a running {Agent}.
+    #
+    class Paused < Action
+    end
+  end
+end

data/lib/spidr/actions/exceptions/skip_link.rb

@@ -0,0 +1,12 @@
+require 'spidr/actions/exceptions/action'
+
+module Spidr
+  module Actions
+    #
+    # An {Actions} exception class which causes a running {Agent} to
+    # skip a link.
+    #
+    class SkipLink < Action
+    end
+  end
+end

data/lib/spidr/actions/exceptions/skip_page.rb

@@ -0,0 +1,12 @@
+require 'spidr/actions/exceptions/action'
+
+module Spidr
+  module Actions
+    #
+    # An {Actions} exception class which causes a running {Agent} to
+    # skip a {Page}, and all links within that page.
+    #
+    class SkipPage < Action
+    end
+  end
+end

data/lib/spidr/actions/exceptions.rb

@@ -0,0 +1,4 @@
+require 'spidr/actions/exceptions/action'
+require 'spidr/actions/exceptions/paused'
+require 'spidr/actions/exceptions/skip_link'
+require 'spidr/actions/exceptions/skip_page'

data/lib/spidr/actions.rb

@@ -0,0 +1,2 @@
+require 'spidrs/actions/exceptions'
+require 'spidrs/actions/actions'