spidr_epg_gem 0.0.0

data/lib/spidr_epg/rules.rb

@@ -0,0 +1,79 @@
+module Spidr
+  #
+  # The {Rules} class represents collections of acceptance and rejection
+  # rules, which are used to filter data.
+  #
+  class Rules
+
+    # Accept rules
+    attr_reader :accept
+
+    # Reject rules
+    attr_reader :reject
+
+    #
+    # Creates a new Rules object.
+    #
+    # @param [Hash] options
+    #   Additional options.
+    #
+    # @option options [Array<String, Regexp, Proc>] :accept
+    #   The patterns to accept data with.
+    #
+    # @option options [Array<String, Regexp, Proc>] :reject
+    #   The patterns to reject data with.
+    #
+    def initialize(options={})
+      @accept = []
+      @reject = []
+
+      @accept += options[:accept] if options[:accept]
+      @reject += options[:reject] if options[:reject]
+    end
+
+    #
+    # Determines whether the data should be accepted or rejected.
+    #
+    # @return [Boolean]
+    #   Specifies whether the given data was accepted, using the rules
+    #   acceptance patterns.
+    #
+    def accept?(data)
+      unless @accept.empty?
+        @accept.any? { |rule| test_data(data,rule) }
+      else
+        !@reject.any? { |rule| test_data(data,rule) }
+      end
+    end
+
+    #
+    # Determines whether the data should be rejected or accepted.
+    #
+    # @return [Boolean]
+    #   Specifies whether the given data was rejected, using the rules
+    #   rejection patterns.
+    #
+    def reject?(data)
+      !accept?(data)
+    end
+
+    protected
+
+    #
+    # Tests the given data against a given pattern.
+    #
+    # @return [Boolean]
+    #   Specifies whether the given data matched the pattern.
+    #
+    def test_data(data,rule)
+      if rule.kind_of?(Proc)
+        rule.call(data) == true
+      elsif rule.kind_of?(Regexp)
+        !((data.to_s =~ rule).nil?)
+      else
+        data == rule
+      end
+    end
+
+  end
+end

data/lib/spidr_epg/sanitizers.rb

@@ -0,0 +1,56 @@
+require 'uri'
+
+module Spidr
+  #
+  # The {Sanitizers} module adds methods to {Agent} which control the
+  # sanitation of incoming links.
+  #
+  module Sanitizers
+    # Specifies whether the Agent will strip URI fragments
+    attr_accessor :strip_fragments
+
+    # Specifies whether the Agent will strip URI queries
+    attr_accessor :strip_query
+
+    #
+    # Sanitizes a URL based on filtering options.
+    #
+    # @param [URI::HTTP, URI::HTTPS, String] url
+    #   The URL to be sanitized
+    #
+    # @return [URI::HTTP, URI::HTTPS]
+    #   The new sanitized URL.
+    #
+    # @since 0.2.2
+    #
+    def sanitize_url(url)
+      url = URI(url.to_s) unless url.kind_of?(URI)
+
+      url.fragment = nil if @strip_fragments
+      url.query    = nil if @strip_query
+
+      return url
+    end
+
+    protected
+
+    #
+    # Initializes the Sanitizer rules.
+    #
+    # @param [Hash] options
+    #   Additional options.
+    #
+    # @option options [Boolean] :strip_fragments (true)
+    #   Specifies whether or not to strip the fragment component from URLs.
+    #
+    # @option options [Boolean] :strip_query (false)
+    #   Specifies whether or not to strip the query component from URLs.
+    #
+    # @since 0.2.2
+    #
+    def initialize_sanitizers(options={})
+      @strip_fragments = options.fetch(:strip_fragments,true)
+      @strip_query     = options.fetch(:strip_query,false)
+    end
+  end
+end

data/lib/spidr_epg/session_cache.rb

@@ -0,0 +1,145 @@
+require 'spidr/spidr'
+
+require 'net/http'
+
+module Spidr
+  #
+  # Stores active HTTP Sessions organized by scheme, host-name and port.
+  #
+  class SessionCache
+
+    # Proxy to use
+    attr_accessor :proxy
+
+    #
+    # Creates a new session cache.
+    #
+    # @param [Hash] proxy (Spidr.proxy)
+    #   Proxy options.
+    #
+    # @option proxy [String] :host
+    #   The host the proxy is running on.
+    #
+    # @option proxy [Integer] :port
+    #   The port the proxy is running on.
+    #
+    # @option proxy [String] :user
+    #   The user to authenticate as with the proxy.
+    #
+    # @option proxy [String] :password
+    #   The password to authenticate with.
+    #
+    # @since 0.2.2
+    #
+    def initialize(proxy=Spidr.proxy)
+      @proxy    = proxy
+      @sessions = {}
+    end
+
+    #
+    # Determines if there is an active HTTP session for a given URL.
+    #
+    # @param [URI::HTTP, String] url
+    #   The URL that represents a session.
+    #
+    # @return [Boolean]
+    #   Specifies whether there is an active HTTP session.
+    #
+    # @since 0.2.3
+    #
+    def active?(url)
+      # normalize the url
+      url = URI(url.to_s) unless url.kind_of?(URI)
+
+      # session key
+      key = [url.scheme, url.host, url.port]
+
+      return @sessions.has_key?(key)
+    end
+
+    #
+    # Provides an active HTTP session for a given URL.
+    #
+    # @param [URI::HTTP, String] url
+    #   The URL which will be requested later.
+    #
+    # @return [Net::HTTP]
+    #   The active HTTP session object.
+    #
+    def [](url)
+      # normalize the url
+      url = URI(url.to_s) unless url.kind_of?(URI)
+
+      # session key
+      key = [url.scheme, url.host, url.port]
+
+      unless @sessions[key]
+        session = Net::HTTP::Proxy(
+          @proxy[:host],
+          @proxy[:port],
+          @proxy[:user],
+          @proxy[:password]
+        ).new(url.host,url.port)
+
+        if url.scheme == 'https'
+          session.use_ssl     = true
+          session.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          session.start
+        end
+
+        @sessions[key] = session
+      end
+
+      return @sessions[key]
+    end
+
+    #
+    # Destroys an HTTP session for the given scheme, host and port.
+    #
+    # @param [URI::HTTP, String] url
+    #   The URL of the requested session.
+    #
+    # @return [nil]
+    #
+    # @since 0.2.2
+    #
+    def kill!(url)
+      # normalize the url
+      url = URI(url.to_s) unless url.kind_of?(URI)
+
+      # session key
+      key = [url.scheme, url.host, url.port]
+
+      if (sess = @sessions[key])
+        begin 
+          sess.finish
+        rescue IOError
+        end
+
+        @sessions.delete(key)
+      end
+    end
+
+    #
+    # Clears the session cache.
+    #
+    # @return [SessionCache]
+    #   The cleared session cache.
+    #
+    # @since 0.2.2
+    #
+    def clear
+      @sessions.each_value do |sess|
+        begin
+          sess.finish
+        rescue IOError
+          nil
+        end
+      end
+
+      @sessions.clear
+      return self
+    end
+
+  end
+end

data/lib/spidr_epg/spidr.rb

@@ -0,0 +1,98 @@
+require 'spidr/agent'
+
+module Spidr
+  # Common proxy port.
+  COMMON_PROXY_PORT = 8080
+
+  # Default proxy information.
+  DEFAULT_PROXY = {
+    :host     => nil,
+    :port     => COMMON_PROXY_PORT,
+    :user     => nil,
+    :password => nil
+  }
+
+  #
+  # Proxy information used by all newly created Agent objects by default.
+  #
+  # @return [Hash]
+  #   The Spidr proxy information.
+  #
+  def Spidr.proxy
+    @@spidr_proxy ||= DEFAULT_PROXY
+  end
+
+  #
+  # Sets the proxy information used by Agent objects.
+  #
+  # @param [Hash] new_proxy
+  #   The new proxy information.
+  #
+  # @option new_proxy [String] :host
+  #   The host-name of the proxy.
+  #
+  # @option new_proxy [Integer] :port (COMMON_PROXY_PORT)
+  #   The port of the proxy.
+  #
+  # @option new_proxy [String] :user
+  #   The user to authenticate with the proxy as.
+  #
+  # @option new_proxy [String] :password
+  #   The password to authenticate with the proxy.
+  #
+  # @return [Hash]
+  #   The new proxy information.
+  #
+  def Spidr.proxy=(new_proxy)
+    @@spidr_proxy = {:port => COMMON_PROXY_PORT}.merge(new_proxy)
+  end
+
+  #
+  # Disables the proxy settings used by all newly created Agent objects.
+  #
+  def Spidr.disable_proxy!
+    @@spidr_proxy = DEFAULT_PROXY
+    return true
+  end
+
+  #
+  # The User-Agent string used by all Agent objects by default.
+  #
+  # @return [String]
+  #   The Spidr User-Agent string.
+  #
+  def Spidr.user_agent
+    @@spidr_user_agent ||= nil
+  end
+
+  #
+  # Sets the Spidr User-Agent string.
+  #
+  # @param [String] new_agent
+  #   The new User-Agent string.
+  #
+  def Spidr.user_agent=(new_agent)
+    @@spidr_user_agent = new_agent
+  end
+
+  #
+  # @see Agent.start_at
+  #
+  def Spidr.start_at(url,options={},&block)
+    Agent.start_at(url,options,&block)
+  end
+
+  #
+  # @see Agent.host
+  #
+  def Spidr.host(name,options={},&block)
+    Agent.host(name,options,&block)
+  end
+
+  #
+  # @see Agent.site
+  #
+  def Spidr.site(url,options={},&block)
+    Agent.site(url,options,&block)
+  end
+end

data/lib/spidr_epg/version.rb

@@ -0,0 +1,4 @@
+module Spidr
+  # Spidr version
+  VERSION = '0.4.1'
+end

data/lib/spidr_epg.rb

@@ -0,0 +1,3 @@
+require 'spidr_epg/agent'
+require 'spidr_epg/spidr'
+require 'spidr_epg/version'

data/lib/spidr_epg_gem.rb~

@@ -0,0 +1,3 @@
+require 'spidr_epg_gem/agent'
+require 'spidr_epg_gem/spidr'
+require 'spidr_epg_gem/version'

data/lib/spidr_epg_gem~

@@ -0,0 +1,7 @@
+module SpidrEpgGem  
+  class WhoIs  
+    def self.awesome?  
+      puts "YOU ARE AWESOME!!"  
+    end  
+  end  
+end  

data/spidr_epg_gem.gemspec

@@ -0,0 +1,17 @@
+Gem::Specification.new do |s|
+  s.platform    = Gem::Platform::RUBY
+  s.name        = 'spidr_epg_gem'
+  s.version     = '0.0.0'
+  s.date = %q{2013-04-15}
+  s.summary = 'Use for crwaling EPG'
+  s.description = 'Use for crwaling EPG'
+  s.required_ruby_version = '>= 1.9.3'
+
+  s.authors     = ["zql"]
+  s.email       = ''
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_path = 'lib'
+  s.requirements << 'none'
+end

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification
+name: spidr_epg_gem
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- zql
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-15 00:00:00.000000000 Z
+dependencies: []
+description: Use for crwaling EPG
+email: ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/spidr.rb
+- lib/spidr_epg.rb
+- lib/spidr_epg/actions.rb
+- lib/spidr_epg/actions/actions.rb
+- lib/spidr_epg/actions/exceptions.rb
+- lib/spidr_epg/actions/exceptions/action.rb
+- lib/spidr_epg/actions/exceptions/paused.rb
+- lib/spidr_epg/actions/exceptions/skip_link.rb
+- lib/spidr_epg/actions/exceptions/skip_page.rb
+- lib/spidr_epg/agent.rb
+- lib/spidr_epg/auth_credential.rb
+- lib/spidr_epg/auth_store.rb
+- lib/spidr_epg/body.rb
+- lib/spidr_epg/cookie_jar.rb
+- lib/spidr_epg/events.rb
+- lib/spidr_epg/extensions.rb
+- lib/spidr_epg/extensions/uri.rb
+- lib/spidr_epg/filters.rb
+- lib/spidr_epg/headers.rb
+- lib/spidr_epg/links.rb
+- lib/spidr_epg/page.rb
+- lib/spidr_epg/rules.rb
+- lib/spidr_epg/sanitizers.rb
+- lib/spidr_epg/session_cache.rb
+- lib/spidr_epg/spidr.rb
+- lib/spidr_epg/version.rb
+- lib/spidr_epg_gem.rb~
+- lib/spidr_epg_gem~
+- spidr_epg_gem.gemspec
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- none
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Use for crwaling EPG
+test_files: []