web-scraper 0.0.5

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+*.log

data/.rspec

@@ -0,0 +1,2 @@
+--format Fuubar
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in web-scraper.gemspec
+gemspec

data/README.md

@@ -0,0 +1,7 @@
+== Web Scraper
+
+Alpha version of a web scraper idea created for an internal use with an 
+abstraction aim. 
+
+In my mind it should be developed further in order to create a dsl to scrape 
+web pages in a simple manner

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/lib/web-scraper/logger_configurer.rb

@@ -0,0 +1,20 @@
+require "logger"
+
+class LoggerConfigurer 
+
+  DEFAULT_FILENAME = 'web-scraper.log'
+
+  def self.create_logger(options = {})        
+    
+    @logger = Logger.new logger_device(options[:log_to_file])    
+    @logger.level      = options[:log_level].to_i if options[:log_level]
+    @logger    
+  end  
+
+  def self.logger_device(file_option)
+    return nil if not file_option and ENV["WEBSCRAPER_ENV"] == 'test'
+
+    file_option ? DEFAULT_FILENAME : STDOUT    
+  end
+
+end

data/lib/web-scraper/scraped_response.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8            
+require "nokogiri"
+
+# == ScrapedResponse
+# Will hold values from the scraping of the page
+class ScrapedResponse
+  # The constructor define at runtime all the methods name 
+  # contained in the key of the hash that will return the computation of the
+  # XPath of the hash value
+  def initialize(page_content, options)     
+    # puts page_content
+    @document = Nokogiri::HTML(page_content)    
+    ScrapedResponse.create_methods(options)
+  end
+  
+  def self.create_methods(options)    
+    options.each do |key,value|      
+      define_method(key) do
+        @document.xpath(value).children.first.text.strip!
+      end            
+    end        
+  end
+end

data/lib/web-scraper/version.rb

@@ -0,0 +1,5 @@
+module Web
+  module Scraper
+    VERSION = "0.0.5"
+  end
+end

data/lib/web_scraper.rb

@@ -0,0 +1,35 @@
+require "web-scraper/version"
+require "web-scraper/logger_configurer" 
+require "web-scraper/scraped_response"
+
+require 'httpclient'
+
+# == Web Scraper
+# 
+module WebScraper
+  # web_page is the method that will start the download type description
+  # options
+  def web_page(base_url, options = {} ,&block)    
+    @logger            = LoggerConfigurer.create_logger(options)
+    @client            = HTTPClient.new
+    @base_url          = base_url  
+
+    block_given? ? self.instance_eval(&block) : self.download(:uri => '/')
+  end
+
+  # download method  
+  def download(params)
+    @logger.info("downloading... #{self.url(params[:uri])}")
+    @logger.debug(" with params... #{params[:params]}")    
+    @last_page = @client.post(self.url(params[:uri]), :body => params[:params], :follow_redirect => true).body
+  end     
+
+  def scrape(options)
+    ScrapedResponse.new(@last_page, options)    
+  end
+
+  def url(uri)
+    "#{@base_url}#{uri}"
+  end
+
+end

data/spec/logger_spec.rb

@@ -0,0 +1,42 @@
+require "spec_helper"
+
+describe LoggerConfigurer do
+
+  before(:all) do
+    ENV['WEBSCRAPER_ENV'] = 'logger_test'
+  end                             
+  
+  after(:all) do
+    ENV['WEBSCRAPER_ENV'] = 'test'
+  end
+
+  it "should create a Logger object w/ no options" do
+    LoggerConfigurer.create_logger.should be_a Logger
+  end
+
+  it "should check for a :log_level option" do
+    LoggerConfigurer.create_logger(:log_level => Logger::INFO).level.should be Logger::INFO
+  end
+
+  context "output to console or file " do
+    before(:each) do           
+      @output         = ''
+      $stdout.stub!( :write ) { |*args| @output.<<( *args )}
+    end
+
+    it "should return a stdout w/o params" do
+      LoggerConfigurer.create_logger.info "testing logger"
+      @output.should include("testing logger")
+    end   
+
+    it "should return stdout for a log_to_file  => false option" do
+      LoggerConfigurer.create_logger(:log_to_file => false).info "testing logger"
+      @output.should include("testing logger")
+    end   
+
+    it "should return FileOutputter for a :file  => true option" do
+      LoggerConfigurer.create_logger(:log_to_file => true).info "testing logger"
+      @output.should be_empty
+    end   
+  end
+end

data/spec/resources/page.html

@@ -0,0 +1,103 @@
+<html>
+<head>
+<style type="text/css">
+
+<!--
+@import "pvgistext.css";
+-->
+</style>
+<title>
+PV power estimate information
+</title>
+<meta http-equiv="Content-Type" content="text/html">
+<meta name="author" content="Created by Thomas Huld - JRC ISPRA - ITALY, Thomas.Huld@jrc.it">
+</head>
+<body text="#000000" bgcolor="#F5F5FF" >
+
+
+<table width="100%" border="0">
+<tr>
+<td align="left" >
+
+<p class=title>
+Performance of Grid-connected PV
+<br>
+</p>
+<p>
+<font color=\"red\"> NOTE: before using these calculations for anything serious, you should read </font><a href="javascript:window.open('PVcalchelp_' + 'en' + '.html','help','height=600,width=600,toolbar=no,scrollbars=yes,resizable'); void 0;"><font color=blue> [this]</font></a>
+</p>
+<p class=subtitle align="left">
+PVGIS estimates of solar electricity generation
+</p><br><font color="red">Warning: the system could not read the horizon file supplied, using standard horizon</font><br>Location: 45°49'3" North,  8°49'34" East, Elevation: 389 m a.s.l.,<br><br>
+</td>
+</tr>
+<tr>
+<td align="left">
+
+<br>
+Solar radiation database used: PVGIS-classic
+<br>
+<br>
+
+Nominal power of the PV system: 1.0 kW (crystalline silicon)<br>
+
+		
+Estimated losses due to temperature: 12.3%  (using local ambient temperature)<br>
+
+Estimated loss due to angular reflectance effects: 3.3%<br>
+Other losses (cables, inverter etc.): 8.0%<br>
+Combined PV system losses: 22.0%
+
+</td>
+</tr>
+<tr>
+<td align="left">
+<br>
+</td>
+</tr>	<tr><td align="left">
+	<table class=data_table border="1" width="300" >
+	<tr>
+	<td colspan=5>
+	<b>Fixed system: inclination=23°, orientation=60° </b>
+	</td>
+	</tr>
+	<tr>
+
+		<th align="left">Month</th><th width=40><span class=formula>E</span><sub>d</sub> </th><th width=40> <span class=formula>E</span><sub>m</sub> </th><th width=40><span class=formula>H</span><sub>d</sub></th><th width=40> <span class=formula>H</span><sub>m</sub> </th></tr>
+<tr> <td> Jan </td><td align="right">1.45</td><td align="right">45.0</td><td align="right">1.71</td><td align="right">53.1</td></tr>
+<tr> <td> Feb </td><td align="right">1.87</td><td align="right">52.5</td><td align="right">2.24</td><td align="right">62.7</td></tr>
+<tr> <td> Mar </td><td align="right">2.90</td><td align="right">90.0</td><td align="right">3.61</td><td align="right">112</td></tr>
+<tr> <td> Apr </td><td align="right">3.32</td><td align="right">99.5</td><td align="right">4.23</td><td align="right">127</td></tr>
+<tr> <td> May </td><td align="right">3.50</td><td align="right">108</td><td align="right">4.59</td><td align="right">142</td></tr>
+<tr> <td> Jun </td><td align="right">4.17</td><td align="right">125</td><td align="right">5.56</td><td align="right">167</td></tr>
+<tr> <td> Jul </td><td align="right">4.31</td><td align="right">134</td><td align="right">5.79</td><td align="right">180</td></tr>
+<tr> <td> Aug </td><td align="right">3.90</td><td align="right">121</td><td align="right">5.26</td><td align="right">163</td></tr>
+<tr> <td> Sep </td><td align="right">3.17</td><td align="right">95.1</td><td align="right">4.15</td><td align="right">124</td></tr>
+<tr> <td> Oct </td><td align="right">2.12</td><td align="right">65.7</td><td align="right">2.66</td><td align="right">82.4</td></tr>
+<tr> <td> Nov </td><td align="right">1.47</td><td align="right">44.1</td><td align="right">1.78</td><td align="right">53.4</td></tr>
+<tr> <td> Dec </td><td align="right">1.28</td><td align="right">39.5</td><td align="right">1.53</td><td align="right">47.3</td></tr><tr><td colspan=5> </td></tr><tr><td><b> Yearly average </b></td><td align="right"><b>2.79 </b></td><td align="right"><b>85.0 </b></td></td><td align="right"><b>3.60 </b></td><td align="right"><b>109 </b></td></tr><tr><td><b>Total for year</b></td><td align="right" colspan=2 ><b>  1020 </b> </td> <td align="right" colspan=2 ><b>  1310 </b> </td> </tr></table></td></tr><tr><td><br></td></tr><tr><td><p>
+<span class=formula>E</span><sub>d</sub>: Average daily electricity production from the given system  (kWh)<br>
+<span class=formula>E</span><sub>m</sub>: Average monthly electricity production from the given system  (kWh)<br>
+<span class=formula>H</span><sub>d</sub>: Average daily sum of global irradiation per square meter received by the modules of the given system  (kWh/m<sup>2</sup>)<br>
+<span class=formula>H</span><sub>m</sub>: Average sum of global irradiation per square meter received by the modules of the given system  (kWh/m<sup>2</sup>)<br></tr></td><tr><td><tr>
+<td>
+<br>
+<p>
+PVGIS &copy; European Communities, 2001-2010<br>
+Reproduction is authorised, provided the source is acknowledged<br>
+See the disclaimer  <a target="legal" href="http://europa.eu/geninfo/legal_notices_en.htm">
+here </a>
+
+
+</p>
+</td>
+</tr>
+</table>
+	<script language="JavaScript">
+		opener.window.setAngles(23,60,1,1);
+	</script	
+	<script language="JavaScript">
+	window.focus();
+	</script>
+	</body>
+	</html>

data/spec/spec_helper.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require 'rubygems'
+
+# This file is copied to spec/ when you run 'rails generate rspec:install'
+ENV["WEBSCRAPER_ENV"] ||= 'test'
+Dir["lib/**/*.rb", "spec/support/*.rb"].each { |f| require File.absolute_path(f) }
+require 'rspec/core'
+          
+def read_resource(filename)
+  IO.read("spec/resources/#{filename}")  
+end
+
+RSpec.configure do |config|
+  # == Mock Framework
+  #
+  # If you prefer to use mocha, flexmock or RR, uncomment the appropriate line:
+  #
+  # config.mock_with :mocha
+  # config.mock_with :flexmock
+  # config.mock_with :rr
+  config.mock_with :rspec
+
+  # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
+  # config.fixture_path = "#{::Rails.root}/spec/fixtures"
+
+  # If you're not using ActiveRecord, or you'd prefer not to run each of your
+  # examples within a transaction, remove the following line or assign false
+  # instead of true.
+  # config.use_transactional_fixtures = true
+
+  # config.before(:suite) do  
+  # end
+  # 
+  # config.after(:suite) do   
+  # end
+
+end

data/spec/support/test_web_server.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+#
+## Bring up server in a new thread (do once?):
+# @mock_server = MockServer.new(4000, 0.5)
+#
+## Pull down server:
+# @mock_server.stop
+#
+## Expectations (rspec example):
+# request_received = false
+# @mock_server.attach do |env|
+# request_received = true
+# env['REQUEST_METHOD'].should == ‘POST’
+# env['PATH_INFO'].should == ‘/foo’
+# [ 200, { 'Content-Type' => 'text/plain', 'Content-Length' => '40' }, [ 'This gets returned from the HTTP request' ]]
+# end
+# request_received.should be_true
+# my_code_that_should_make_post_request # to http://localhost:4000/foo
+#
+## After each test:
+# @mock_server.detach
+require "WEBrick"
+require 'rack'     
+
+class TestWebServer
+  def initialize(port = 4000, pause = 0.5)
+    @block = nil
+    @parent_thread = Thread.current
+    @thread = Thread.new do
+      Rack::Handler::WEBrick.run(self, :Port => port, 
+        :Logger => WEBrick::Log.new("/dev/null"), 
+        :AccessLog => [nil, nil])
+    end
+    sleep pause # give the server time to fire up… YUK!
+  end
+  
+  def stop                  
+    Thread.kill(@thread)
+  end
+  
+  def attach(&block)
+    @block = block
+  end
+  
+  def detach()
+    @block = nil
+  end
+  
+  def call(env)
+    begin
+      raise "Specify a handler for the request using attach(block), the block should return a valid rack response and can test expectations" unless @block
+      @block.call(env)
+    rescue Exception => e
+      @parent_thread.raise e
+      [ 500, { 'Content-Type' => 'text/plain', 'Content-Length' => '13' }, [ 'Bad test code' ]]
+    end
+  end      
+
+  # body is the content returned                           
+  # :method can be :get - default or :post]
+  # :path => '/pippo'
+  def stub_response(body, options = {})
+    method = options[:method] ? options[:method].to_s.upcase : 'GET'
+    path = options[:path] ? options[:path] : '/'
+    content_type = options[:content_type] ? options[:content_type] : 'text/plain'
+                          
+    attach do |env|  
+      env['REQUEST_METHOD'].should == method
+      env['PATH_INFO'].should == path
+      [ 200, { 'Content-Type' => content_type, 'Content-Length' => body.length.to_s }, [ body ]]
+    end
+  end
+    
+end

data/spec/web_scraper_spec.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe WebScraper do
+
+  before(:each) do
+    extend WebScraper
+  end
+
+  context "[w/o web server]" do   
+    it "should be raised an error when calling download method w/o :uri param" do
+      expect { download }.to raise_error        
+    end
+  end
+  
+  context "[w/ web server]" do
+    before(:all) do
+      $web_server = TestWebServer.new      
+    end    
+    
+    context "starting defining dsl" do
+
+      before(:each) do
+        $web_server.stub_response('Root Page', {:method => :post, :path => '/'})
+      end
+
+      it "should accept a url and return its content" do
+        page = web_page "http://localhost:4000"
+        page.should eq 'Root Page'
+      end    
+
+      it "should execute a block as a param and return its content" do
+        page = web_page "http://localhost:4000" do
+          "test"
+        end
+        page.should eq "test"
+      end    
+      
+    end
+
+    context "download a page" do
+      before(:each) do
+        $web_server.stub_response('Web Page Scraped', {:method => :post, :path => '/foo'})
+      end
+
+      it "should download a page" do
+        page = web_page "http://localhost:4000" do
+          download :uri => "/foo"
+        end
+        page.should eq "Web Page Scraped"    
+      end                     
+      
+    end
+    
+    context "scrape a page" do
+      before(:each) do
+        $web_server.stub_response(read_resource('page.html'), {:method => :post, :path => '/page.html'})
+      end
+
+      it "should scrape content from a downloaded of page" do
+        page = web_page "http://localhost:4000" do
+          download :uri => "/page.html"
+          scrape :avg_monthly_production => '//table/tr[4]/td/table/tr[17]/td[2]/b'
+        end
+        page.avg_monthly_production.should eq "1020"
+      end                     
+
+    end
+    
+    after(:each) do
+      $web_server.detach            
+    end                        
+    
+    after(:all) do
+      $web_server.stop
+    end                        
+        
+  end
+end

data/web-scraper.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "web-scraper/version"
+
+Gem::Specification.new do |s|
+  s.name        = "web-scraper"
+  s.version     = Web::Scraper::VERSION
+  s.authors     = ["Gian Carlo Pace"]
+  s.email       = ["giancarlo.pace@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Web Values scraper}
+  s.description = %q{It's an utility to scrape web pages}
+
+  s.rubyforge_project = "web-scraper"
+
+  s.add_dependency "httpclient", "~> 2.2.1"
+  s.add_dependency "rspec"
+  s.add_dependency "rack"
+  s.add_dependency "nokogiri"
+  s.add_dependency "logger"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: web-scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.5
+  prerelease: 
+platform: ruby
+authors:
+- Gian Carlo Pace
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-07-07 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: &2160925440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *2160925440
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2160925000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2160925000
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: &2160924540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2160924540
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &2160924120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2160924120
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: &2160923700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2160923700
+description: It's an utility to scrape web pages
+email:
+- giancarlo.pace@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- README.md
+- Rakefile
+- lib/web-scraper/logger_configurer.rb
+- lib/web-scraper/scraped_response.rb
+- lib/web-scraper/version.rb
+- lib/web_scraper.rb
+- spec/logger_spec.rb
+- spec/resources/page.html
+- spec/spec_helper.rb
+- spec/support/test_web_server.rb
+- spec/web_scraper_spec.rb
+- web-scraper.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: web-scraper
+rubygems_version: 1.8.5
+signing_key: 
+specification_version: 3
+summary: Web Values scraper
+test_files:
+- spec/logger_spec.rb
+- spec/resources/page.html
+- spec/spec_helper.rb
+- spec/support/test_web_server.rb
+- spec/web_scraper_spec.rb