spidr 0.5.0 → 0.6.0

data/lib/spidr/page/body.rb

@@ -1,98 +0,0 @@
-require 'nokogiri'
-
-module Spidr
-  class Page
-    #
-    # The body of the response.
-    #
-    # @return [String]
-    #   The body of the response.
-    #
-    def body
-      (response.body || '')
-    end
-
-    #
-    # Returns a parsed document object for HTML, XML, RSS and Atom pages.
-    #
-    # @return [Nokogiri::HTML::Document, Nokogiri::XML::Document, nil]
-    #   The document that represents HTML or XML pages.
-    #   Returns `nil` if the page is neither HTML, XML, RSS, Atom or if
-    #   the page could not be parsed properly.
-    #
-    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Document.html
-    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html
-    #
-    def doc
-      unless body.empty?
-        begin
-          if html?
-            @doc ||= Nokogiri::HTML(body, @url.to_s, content_charset)
-          elsif (rss? || atom? || xml? || xsl?)
-            @doc ||= Nokogiri::XML(body, @url.to_s, content_charset)
-          end
-        rescue
-        end
-      end
-    end
-
-    #
-    # Searches the document for XPath or CSS Path paths.
-    #
-    # @param [Array<String>] paths
-    #   CSS or XPath expressions to search the document with.
-    #
-    # @return [Array]
-    #   The matched nodes from the document.
-    #   Returns an empty Array if no nodes were matched, or if the page
-    #   is not an HTML or XML document.
-    #
-    # @example
-    #   page.search('//a[@href]')
-    #
-    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Node.html#M000239
-    #
-    def search(*paths)
-      if doc
-        doc.search(*paths)
-      else
-        []
-      end
-    end
-
-    #
-    # Searches for the first occurrence an XPath or CSS Path expression.
-    #
-    # @return [Nokogiri::HTML::Node, Nokogiri::XML::Node, nil]
-    #   The first matched node. Returns `nil` if no nodes could be matched,
-    #   or if the page is not a HTML or XML document.
-    #
-    # @example
-    #   page.at('//title')
-    #
-    # @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/XML/Node.html#M000251
-    #
-    def at(*arguments)
-      if doc
-        doc.at(*arguments)
-      end
-    end
-
-    alias / search
-    alias % at
-
-    #
-    # The title of the HTML page.
-    #
-    # @return [String]
-    #   The inner-text of the title element of the page.
-    #
-    def title
-      if (node = at('//title'))
-        node.inner_text
-      end
-    end
-
-    alias to_s body
-  end
-end

data/spec/helpers/history.rb

@@ -1,34 +0,0 @@
-module Helpers
-  module History
-    def visited_once?(url)
-      return @agent.visited_urls.select { |visited_url|
-        visited_url == url
-      }.length == 1
-    end
-
-    def visited_link?(url)
-      @agent.visited?(url)
-    end
-
-    def visit_failed?(url)
-      @agent.failed?(url)
-    end
-
-    def should_visit_link(url)
-      expect(visited_link?(url)).to eq(true)
-    end
-
-    def should_ignore_link(url)
-      expect(visited_link?(url)).to eq(false)
-    end
-
-    def should_visit_once(url)
-      expect(visited_once?(url)).to eq(true)
-    end
-
-    def should_fail_link(url)
-      expect(visited_link?(url)).to eq(false)
-      expect(visit_failed?(url)).to eq(true)
-    end
-  end
-end

data/spec/helpers/page.rb

@@ -1,8 +0,0 @@
-require 'net/http'
-require 'uri'
-
-def get_page(url)
-  url = URI(url.to_s)
-
-  return Spidr::Page.new(url,Net::HTTP.get_response(url))
-end

data/spec/helpers/wsoc.rb

@@ -1,83 +0,0 @@
-require 'wsoc/config'
-require 'open-uri'
-require 'json'
-
-require 'helpers/history'
-
-module Helpers
-  module WSOC
-    include History
-
-    SERVER_URL = URI::HTTP.build(
-      host: (ENV['HOST'] || ::WSOC::Config::DEFAULT_HOST),
-      port: (ENV['PORT'] || ::WSOC::Config::DEFAULT_PORT)
-    )
-
-    SPECS_URL = SERVER_URL.merge(::WSOC::Config::SPECS_PATHS[:json])
-
-    COURSE_URL = SERVER_URL.merge(::WSOC::Config::COURSE_START_PATH)
-
-    COURSE_METADATA = {}
-
-    def self.included(base)
-      hash = JSON.parse(open(SPECS_URL).read)
-      metadata = hash['metadata']
-      specs = hash['specs']
-
-      if metadata.kind_of?(Hash)
-        COURSE_METADATA.merge!(metadata)
-      end
-
-      if specs.kind_of?(Array)
-        specs.each do |spec|
-          message = spec['message'].dump
-          url = spec['url'].dump
-
-          case spec['behavior']
-          when 'visit'
-            base.module_eval %{
-              it #{message} do
-                should_visit_link(#{url})
-              end
-            }
-          when 'ignore'
-            base.module_eval %{
-              it #{message} do
-                should_ignore_link(#{url})
-              end
-            }
-          when 'fail'
-            base.module_eval %{
-              it #{message} do
-                should_fail_link(#{url})
-              end
-            }
-          end
-        end
-      end
-    end
-
-    def course
-      WSOC::COURSE_METADATA
-    end
-
-    def course_auth_store
-      course['auth_store']
-    end
-
-    def run_course
-      Spidr::Agent.start_at(COURSE_URL) do |agent|
-        course_auth_store.each do |path,auth|
-          agent.authorized.add(
-            COURSE_URL.merge(path),
-            auth['user'],
-            auth['password']
-          )
-        end
-
-        agent.every_failed_url { |url| puts "[FAILED] #{url}" }
-        agent.every_url { |url| puts url }
-      end
-    end
-  end
-end

data/spec/page_examples.rb

@@ -1,21 +0,0 @@
-require 'spidr/page'
-
-require 'spec_helper'
-
-shared_examples_for "Page" do
-  it "should have a status code" do
-    expect(@page.code).to be_integer
-  end
-
-  it "should have a body" do
-    expect(@page.body).not_to be_empty
-  end
-
-  it "should provide transparent access to the response headers" do
-    expect(@page.content_type).to eq(@page.response['Content-Type'])
-  end
-
-  it "should allow content-types" do
-    expect(@page.content_types).not_to be_empty
-  end
-end