scrapes 0.2.0

data/lib/scrapes/to_proxy.rb

@@ -0,0 +1,50 @@
+################################################################################
+#
+# Copyright (C) 2006 Peter J Jones (pjones@pmade.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+################################################################################
+module Scrapes
+  ################################################################################
+  # Link pages together.  Useful for when one only contains links to the next page.
+  class ToProxy
+    ################################################################################
+    def initialize (from, to)
+      @from, @to = from, to
+    end
+
+    ################################################################################
+    def extract (data, uri, session, &block)
+      @from.extract(data, uri, session) do |link|
+        session.page(@to, link, &block)
+      end
+    end
+
+    ################################################################################
+    # Continue the string of connections.
+    def to (next_in_line)
+      self.class.new(self, next_in_line)
+    end
+
+  end
+  ################################################################################
+end
+################################################################################

data/test/cache.rb

@@ -0,0 +1,75 @@
+################################################################################
+#
+# Copyright (C) 2006 Peter J Jones (pjones@pmade.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+################################################################################
+require 'fileutils'
+require 'test/lib/server'
+require 'scrapes/cache'
+require 'test/unit'
+
+class TestCache < Test::Unit::TestCase
+  include LocalHTTPServer
+
+  def setup
+    start_server
+    @cache = Scrapes::Cache.new
+  end
+
+  def teardown
+    stop_server
+    FileUtils.remove_dir 'cache' if File.exist? 'cache'
+  end
+
+  def test_truth
+    assert @server
+    assert @cache
+  end
+
+  def test_directory_attr
+    assert_equal @cache.directory, File.expand_path('cache')
+    assert_nothing_raised { @cache.directory = 'cache' }
+    assert_equal @cache.directory, 'cache'
+  end
+
+  def test_enabled_attr
+    assert_equal @cache.enabled, false
+    assert_nothing_raised { @cache.enabled = true }
+    assert @cache.enabled
+  end
+
+  def test_update
+    assert_nothing_raised { @cache.update 'foo.txt', localhost_http_get('foo.txt') }
+    assert(!@cache.check('foo.txt'))
+    assert_nothing_raised { @cache.enabled = true }
+    assert_nothing_raised { @cache.update 'foo.txt', localhost_http_get('foo.txt') }
+    assert(@cache.check('foo.txt'))
+  end
+
+  def test_without_cache
+    assert_nothing_raised { @cache.enabled = true }
+    @cache.without_cache do
+      assert_nothing_raised { @cache.update 'foo.txt', localhost_http_get('foo.txt') }
+      assert(!@cache.check('foo.txt'))
+    end
+  end
+end

data/test/cookies.rb

@@ -0,0 +1,34 @@
+################################################################################
+#
+# Copyright (C) 2006 Peter J Jones (pjones@pmade.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+################################################################################
+require 'scrapes/cookies'
+require 'test/unit'
+
+class TestCookies < Test::Unit::TestCase
+  def test_parser
+    cookies = Scrapes::Cookies.new
+    cookies.from_header('sid=21;domain=.example.com;Path=/')
+    assert_equal('sid=21', cookies.to_header)
+  end
+end

data/test/crawler.rb

@@ -0,0 +1,69 @@
+################################################################################
+#
+# Copyright (C) 2006 Peter J Jones (pjones@pmade.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+################################################################################
+require 'fileutils'
+require 'test/lib/server'
+require 'scrapes/crawler'
+require 'scrapes/session'
+require 'test/unit'
+
+class TestCrawler < Test::Unit::TestCase
+  include LocalHTTPServer
+
+  def setup
+    @session = Scrapes::Session::new
+    start_server
+    @crawler = Scrapes::Crawler.new @session
+  end
+
+  def teardown
+    stop_server
+    FileUtils.remove_dir 'cache' if File.exist? 'cache'
+  end
+
+  def test_truth
+    assert @session
+    assert @server
+    assert @crawler
+  end
+
+  def test_cache_attr
+    assert @crawler.cache
+    cache = Scrapes::Cache.new
+    assert_nothing_raised { @crawler.cache = cache }
+    assert_equal @crawler.cache, cache
+  end
+
+  def test_log_attr
+    assert @crawler.log.nil?
+    log = Object.new
+    assert_nothing_raised { @crawler.log = log }
+    assert_equal @crawler.log, log
+  end
+
+  def test_fetch
+    assert @crawler.fetch(localhost_url('foo.txt'))
+    assert_equal @crawler.fetch(localhost_url('dummy')).class, Net::HTTPNotFound
+  end
+end

data/test/hpricot.rb

@@ -0,0 +1,55 @@
+################################################################################ #
+# Copyright (C) 2006 Peter J Jones (pjones@pmade.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+################################################################################
+require 'rubygems'
+################################################################################
+require 'scrapes'
+################################################################################
+require 'test/lib/server'
+require 'test/unit'
+
+class TestRedhandedPage < Test::Unit::TestCase
+  include LocalHTTPServer
+
+  def setup
+    start_server
+    Scrapes::Initializer.run do |initializer|
+      initializer.pages_parent = 'test'
+      initializer.process
+    end
+  end
+
+  def teardown
+    stop_server
+  end
+
+  def test_truth
+    assert @server
+  end
+
+  def test_texts
+    Scrapes::Session.start do |session|
+      @page = session.page(LocalRedhanded, localhost_url('redhanded.html'))
+    end
+  end
+end

data/test/initializer.rb

@@ -0,0 +1,54 @@
+################################################################################
+#
+# Copyright (C) 2006 Peter J Jones (pjones@pmade.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+################################################################################
+require 'rubygems'
+require 'scrapes/initializer'
+require 'scrapes/page'
+require 'test/unit'
+
+class TestInitializer < Test::Unit::TestCase
+  def setup
+    assert_nothing_raised { @initializer = Scrapes::Initializer.new }
+  end
+
+  def test_run
+    assert_nothing_raised { Scrapes::Initializer.run { } }
+  end
+
+  def test_pages_parent
+    assert_equal @initializer.pages_parent, File.dirname($0)
+    assert_nothing_raised { @initializer.pages_parent = 'foobar' }
+    assert_equal @initializer.pages_parent, 'foobar'
+  end
+
+  def test_pages_dir
+    assert_equal @initializer.pages_dir, 'pages'
+    assert_nothing_raised { @initializer.pages_dir = 'foobar' }
+    assert_equal @initializer.pages_dir, 'foobar'
+  end
+
+  def test_process
+    assert_nothing_raised { @initializer.process }
+  end
+end

data/test/lib/server.rb

@@ -0,0 +1,63 @@
+################################################################################
+#
+# Copyright (C) 2006 Peter J Jones (pjones@pmade.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+################################################################################
+# TODO figure out how to suppress get output
+################################################################################
+require 'logger'
+require 'webrick'
+require 'net/http'
+################################################################################
+# webrick localhost http server
+module LocalHTTPServer
+  ################################################################################
+  # start the server and return it
+  def start_server
+    @server = WEBrick::HTTPServer.new :Port=>4270, :Logger=>Logger.new(nil),
+      :DocumentRoot=>File.expand_path('test/public'), :AccessLog=>[]
+    @server_thread = Thread.new { @server.start }
+  end
+
+  ################################################################################
+  # wait for server to shutdown and return it
+  def stop_server
+    if @server
+      @server.shutdown
+      @server_thread.join
+      @server
+    end
+  end
+
+  ################################################################################
+  # return a localhost url given a doc path
+  def localhost_url path = nil
+    "http://localhost:4270/#{path}"
+  end
+
+  ################################################################################
+  # get a page from the localhost http server
+  def localhost_http_get path = nil
+    Net::HTTP.get(URI.parse(localhost_url(path)))
+  end
+end
+################################################################################

data/test/page.rb

@@ -0,0 +1,77 @@
+################################################################################ #
+# Copyright (C) 2006 Peter J Jones (pjones@pmade.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+################################################################################
+require 'rubygems'
+################################################################################
+require 'scrapes'
+################################################################################
+require 'test/lib/server'
+require 'test/unit'
+
+class TestRedhandedPage < Test::Unit::TestCase
+  include LocalHTTPServer
+
+  def setup
+    start_server
+    Scrapes::Initializer.run do |initializer|
+      initializer.pages_parent = 'test'
+      initializer.process
+    end
+  end
+
+  def teardown
+    stop_server
+  end
+
+  def test_truth
+    assert @server
+  end
+
+  def test_local_redhanded
+    Scrapes::Session.start do |session|
+      @page = session.page(LocalRedhanded, localhost_url('redhanded.html'))
+    end
+    assert_equal  Array          , @page.syndicate_content.class
+    assert_equal  2              , @page.syndicate_content.size 
+    assert_equal  '/index.xml'   , @page.syndicate_link         
+    assert_equal  'JavaScript'   , @page.script_language        
+    assert_equal  274            , @page.links.size
+    #assert_equal  0              , @page.element.size
+    assert_equal  'RSS'          , @page.syndicate_content[0]   
+    assert_equal  '2.0'          , @page.syndicate_content[1]   
+  end
+
+  def test_local_redhanded_enties
+    Scrapes::Session.start do |session|
+      @entries = session.page(LocalRedhandedEntries, localhost_url('redhanded.html'))
+    end
+    assert_equal 20, @entries.size
+    #assert_equal "Denver Accord#", @entries[0].entry_title
+  end
+
+  def test_local_pagination_1
+    Scrapes::Session.start do |session|
+      @foil = session.page(LocalPagination, localhost_url('foil74.html'))
+    end
+  end
+end