korfzone-scraper 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 92440e35b72111385ff8bd5a4ff54f5968593e43
+  data.tar.gz: 2349511ac04326bc50f420183b67ed2ed54a1e8e
+SHA512:
+  metadata.gz: 09532104cc8a7b680b7a56204eba9137501c541767e0bb7b9ee1466bb12d3c27194a91358a4e00540904075850075ccf9ca3a1932e7d70ebc71f406e2d20b978
+  data.tar.gz: 84cb5e0cc1f2bfd96dc84d0c973f87059229f58ed56bf811465fe19009d6babc8731bfa3b89572960b2aa12c9316e4e4942fc817cd504d8c77591d09075b657f

data/.gitignore

@@ -0,0 +1,5 @@
+Gemfile.lock
+.yardoc
+doc
+coverage
+.DS_Store

data/.simplecov

@@ -0,0 +1,3 @@
+SimpleCov.start do
+  add_filter "/spec/"
+end

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in korfzone-scraper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Rutger Claes
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Korfzone::Scraper
+
+Using this scraper you can extract all information about korfball games from the korfbal.be website.
+
+The game information extracted using this tool is not linked.  For example, to list all games for one club you will have to map all team names to a club.  The [Korfzone API](http://korfzone.be/api) offers a linked version of the data scraped from the KBKB website.  It also hosts data from previous seasons.
+
+This project is extracted from the Korfzone code base.  The korfbal.be website uses very little to no semantic HTML.  The scraper is therefore very brittle.  If the developers decide to add a column to the games table or they change the class of the rows, this code will break.
+
+Korfzone and the Korfzone Scraper are not affiliated with the KBKB in any way.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'korfzone-scraper'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install korfzone-scraper
+
+## Usage
+
+The scraper has two main functions: Firstly finding the uri's to all pages containing the game information and secondly extracting that information from those pages.  The korfbal.be website does not offer an easy to use single point of entry from which to scrape all uri's.  The links to the relevant pages themselves are also scattered across a number of pages.  Below are examples on how to find the relevant uri's and how to extract games from a single uri.
+
+### Scraping the uri's to all pages for a certain category
+
+```ruby
+entry_points = Korfzone::Scraper::Page.for_category :senioren
+entry_points.each do |page|
+  puts page.block_uris.map { |uri| uri.to_s }.join( "\n" )
+end
+```
+
+### Scraping the games of an individual page
+
+```ruby
+page = Korfzone::Scraper::Page.new 'http://www.korfbal.be/beta/Wedstrijden/senioren/veld/V01'
+page.games do |game|
+  puts game[ :starts_at ]
+  puts game[ :teams ].join( ' - ' )
+  puts game[ :location ]
+  puts "=" * 80
+end
+```
+
+## ToDo list
+
+1. Handle errors.  The korfbal.be site occasionally returns status code 500.  The scraper should handle this.
+2. Check support for E-tags on the korfbal.be website.  If they are supported, it would make sense to support them in the scraper as well.
+3. Write a binary, probably based on Thor to facilitate scraping.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+Consider also contributing to the overall [Korfzone project](http://github.org/korfzone).

data/Rakefile

@@ -0,0 +1,4 @@
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)

data/korfzone-scraper.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'korfzone/scraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "korfzone-scraper"
+  spec.version       = Korfzone::Scraper::VERSION
+  spec.authors       = ["Rutger Claes"]
+  spec.email         = ["rutger@korfzone.be"]
+  
+  spec.description   = %q{Scrape the KBKB korfbal.be website}
+  spec.summary       = %q{The code needed to scrape all games of the KBKB website (korfbal.be) extracted from the Korfzone project.  This code is expected to break whenever the KBKB updates its website.}
+  
+  spec.homepage      = "http://github.org/korfzone/korfzone-scraper"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^spec/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "debugger"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "simplecov"
+end

data/lib/korfzone/scraper/block_page.rb

@@ -0,0 +1,39 @@
+module Korfzone
+
+  module Scraper
+
+    # Code to handle links to different block pages.
+    #
+    # Games on the {korfbal.be} website are divided into blocks.  Every block
+    # has its own page.  At the top of every page links to every other block 
+    # can be found.
+    module BlockPage
+
+      # Extracts the uri's of all block pages found at the top of this page.
+      #
+      # @return [Array<URI>] An array containing all uri's found on this page
+      # @yieldparam [URI] An uri found on this page
+      def block_uris
+        Array.new.tap do |block_urls|
+          document.css( '.block @href' ).each do |block_element|
+            block_url = resolve_uri block_element.value
+            yield block_url if block_given?
+            block_urls << block_url
+          end
+        end
+      end
+
+      # Extracts the uri's of all block pages found at the top of this page 
+      # encapsulated in {Korfzone::Scraper::Page} objects.
+      #
+      # @return [Array<Page>] An array containing other block pages
+      # @yieldparam [Page] A page containing games for one block
+      def block_pages
+        block_uris.map do |uri| 
+          Page.new( uri.to_s ).tap { |uri| yield uri if block_given? }
+        end
+      end
+
+    end
+  end
+end

data/lib/korfzone/scraper/categories.rb

@@ -0,0 +1,26 @@
+module Korfzone
+
+  module Scraper
+
+    # Starting uri's for all of the competitions
+    CATEGORIES = {
+      senioren: {
+        zaal: 'http://korfbal.be/beta/Wedstrijden/senioren/zaal',
+        veld: 'http://korfbal.be/beta/Wedstrijden/senioren/veld',
+        beker: 'http://korfbal.be/beta/Wedstrijden/senioren/beker senioren'
+      },
+      jeugd: {
+        veldvoorronde: 'http://korfbal.be/beta/Wedstrijden/jeugd/veldvoorronde',
+        zaal: 'http://korfbal.be/beta/Wedstrijden/jeugd/zaal',
+        veldeindronde: 'http://korfbal.be/beta/Wedstrijden/jeugd/veldeindronde',
+        beker: 'http://korfbal.be/beta/Wedstrijden/jeugd/beker'
+      },
+      gewestelijken: {
+        veld: 'http://korfbal.be/beta/Wedstrijden/gewestelijken/veld',
+        zaal: 'http://korfbal.be/beta/Wedstrijden/gewestelijken/zaal'
+      }
+    }
+
+  end
+
+end

data/lib/korfzone/scraper/games_page.rb

@@ -0,0 +1,112 @@
+module Korfzone
+
+  module Scraper
+
+    # Code to extract all games from one page
+    module GamesPage
+
+      def category
+        extract_category_and_competition if @category.nil?
+        @category
+      end
+
+      def competition
+        extract_category_and_competition if @competition.nil?
+        @competition
+      end
+
+      def games
+        Array.new.tap do |games|
+          rows = document.xpath( '//*[@id="ContentPlaceHolder1_UpdatePanel1"]/table[2]/tr[td[ contains( @class, "gameLeagueHeader" ) or contains( @class, "rptItem" ) or contains( @class, "rptAltItem" )]]' )
+          rows.inject( nil ) do |division,row|
+            if division.nil? || row.css( 'td.gameLeagueHeader' ).any?
+              row.text.strip.gsub( /\s+/, ' ' ) 
+            else
+              division.tap do |division|
+                attributes = parse_game_row( row )
+                unless attributes.nil?
+                  attributes.merge!( division: division, category: category.to_s, competition: competition )
+                  yield attributes if block_given?
+                  games << attributes
+                end # end unless attributes nil
+              end # end division tap
+            end # if division.nil?
+          end # end inject
+        end # end games tap
+      end # end games
+
+      private
+
+      CATEGORY_ID = 'ContentPlaceHolder1_lblTitle'
+
+      def extract_category_and_competition
+        value = document.css( "##{CATEGORY_ID}" ).first.text
+        values = value.split( '-', 2 ).map { |s| s.strip }
+        
+        @category = values.first.gsub( /^wedstrijden /i, '' ).to_sym
+        @competition = values.last
+
+        [ @category, @competition ]
+      end
+
+      def parse_game_row( row )
+        mapping = case row.css( 'td' ).count
+                  when 10
+                    [ :code, nil, :date, :time, :home_team, :visiting_team, nil, :referee, nil, :location ]
+                  when 11
+                    [ :code, nil, :date, :time, :home_team, :visiting_team, :home_score, :visiting_score, :referee, nil, :location ]
+                  else
+                    raise "Unparseable row encountered: #{row.to_s}"
+                  end # end case
+
+        values = row.css( 'td' ).map do |td|
+          td.css( 'a' ).any? ? [ td.css( 'a' ).first.text, resolve_uri( td.css( 'a' ).first[ 'href' ] ) ] : td.text
+        end
+
+        values = mapping.zip values
+        values.delete_if { |key,value| key.nil? }
+        values = Hash[ values ]
+        game_is_complete?( values ) ? cleanup_game_attributes( values ) : nil
+      end #end parse_game_row
+
+      def game_is_complete?( data )
+        [ :home_team, :visiting_team, :date, :time ].all? { |k| !data[ k ].nil? && data[k].to_s.strip != '' }
+      end
+
+      def cleanup_game_attributes( data )  
+        data.clone.tap do |data|
+          # Fixing the date
+          day, month, year = data.delete( :date ).split( '/', 3 ).map { |i| i.to_i }
+          hour, minutes = data.delete( :time ).split( ':', 2 ).map { |i| i.to_i }
+          data[ :starts_at ] = Time.local( year, month, day, hour, minutes, 0 )
+
+          # Cleanup the teams and the game code
+          [ :home_team, :visiting_team, :code, :referee ].each do |key|
+            data[ key ] = data[ key ].strip.gsub( /\s+/, ' ' )
+          end
+
+          # Cleanup the location name
+          if data.key?( :location )
+            data[ :location ][0] = data[ :location ][0].strip.gsub( /\s+/, ' ' )
+          end
+
+          # Combine the teams
+          data[ :teams ] = [ data.delete( :home_team ), data.delete( :visiting_team ) ]
+
+          # Combine the scores if there are any
+          if data.key?( :home_score )
+            forfait = data[ :visiting_score ] =~ /ff/i
+            home_score = data.delete( :home_score ).gsub( /[^0-9]+/, '' ).to_i
+            visiting_score = data.delete( :visiting_score ).gsub( /[^0-9]+/, '' ).to_i
+            data[ :scores ] = [ home_score, visiting_score ]
+            data[ :forfait ] = true if forfait
+          end
+
+          # Remove empty fields
+          [ :referee, :location ].each { |key| data.delete( key ) if data[ key ].nil? || data[ key ] =~ /^\s*$/ }
+        end # end tap
+      end #end cleanup_game_attributes
+
+    end
+  end
+end

data/lib/korfzone/scraper/page.rb

@@ -0,0 +1,47 @@
+require 'nokogiri'
+require 'uri'
+
+require 'korfzone/scraper/block_page'
+require 'korfzone/scraper/games_page'
+
+module Korfzone
+
+  module Scraper
+
+    class Page
+
+      include BlockPage
+      include GamesPage
+
+      attr_accessor :url
+
+      def initialize( url )
+        @url = URI( URI.escape url.to_s )
+      end
+
+      def document
+        @document ||= Nokogiri::HTML( ::Korfzone::Scraper.fetch url )
+      end
+
+      def ==( other_page )
+        other_page.respond_to?( :url ) && other_page.url == url
+      end
+
+      protected
+
+      def resolve_uri( uri )
+        uri if uri.to_s =~ /^http/
+        @url.clone.tap do |resolved_uri|
+          resolved_uri.path = URI.escape( uri.to_s )
+        end
+      end
+
+      def self.for_category( category )
+        ::Korfzone::Scraper::CATEGORIES[ category ].values.map { |url| Page.new url }
+      end
+
+    end
+
+  end
+
+end

data/lib/korfzone/scraper/version.rb

@@ -0,0 +1,5 @@
+module Korfzone
+  module Scraper
+    VERSION = "1.0.0"
+  end
+end

data/lib/korfzone/scraper.rb

@@ -0,0 +1,38 @@
+require "korfzone/scraper/version"
+
+require 'open-uri'
+
+module Korfzone
+
+  # The module containing all scraping related code.
+  #
+  # The games on the korfbal.be website are scattered across a number of pages.  A first division is made
+  # on the category of the game ('senioren', 'gewestelijken' or 'jeugd').  A second division is made on the
+  # competition of the game ('veld', 'zaal', 'beker', ... ).  The final division is into blocks based on the day
+  # of the match.
+  #
+  # There is no simple way to scrape the uri's to the different pages that can act as starting point to scrape a competition,
+  # therefore the starting points are listed explicitly in {Korfzone::Scraper::CATEGORIES}.  The pages listed there
+  # have a link to all blocks for that competition.
+  # 
+  # * {Korfzone::Scraper::Page} is the main class.  It represents a single scrapeable page on korfbal.be.
+  # * {Korfzone::Scraper::BlockPage} contains all functionality to extract the different game day pages.
+  # * {Korfzone::Scraper::GamesPage} contains all functionality to extract games from a page.
+  #
+  # @author Rutger Claes <rutger@korfzone.be>
+  module Scraper
+
+    # Method to fetch content, introduced to improve testability
+    #
+    # @param uri [#to_s] The uri to fetch
+    # @return the contents of the uri
+    def self.fetch( uri )
+      open( uri.to_s )
+    end
+
+  end
+
+end
+
+require 'korfzone/scraper/categories'
+require 'korfzone/scraper/page'

data/spec/korfzone/scraper/page_spec.rb

@@ -0,0 +1,224 @@
+require 'spec_helper'
+
+describe Korfzone::Scraper::Page do
+
+  it "should download and parse its uri" do
+    uri  = 'http://www.korfbal.be/beta/Wedstrijden/senioren/veld'
+    stub_korfzone_fetch uri
+    page = Korfzone::Scraper::Page.new uri
+    expect( page.document ).not_to be_nil
+  end
+
+  describe "containing links to other blocks" do
+
+    it "should list all blocks as uri" do
+      uri = 'http://www.korfbal.be/beta/Wedstrijden/senioren/veld'
+      stub_korfzone_fetch uri
+      page = Korfzone::Scraper::Page.new uri
+      expect( page.block_uris ).to eq( BLOCK_URIS )
+    end
+
+    it "should list all blocks as page" do
+      uri = 'http://www.korfbal.be/beta/Wedstrijden/senioren/veld'
+      stub_korfzone_fetch uri
+      page = Korfzone::Scraper::Page.new uri
+      expect( page.block_pages ).to eq( BLOCK_URIS.map { |u| Korfzone::Scraper::Page.new u } )
+    end
+
+  end
+
+  shared_examples_for "successful scraping" do
+
+    it "should load the dom tree" do
+      expect( page.document.to_s ).to match( /#{code}/i )
+    end
+
+  end
+
+
+  shared_examples_for "basic game" do
+
+    it "should extract the game" do
+      expect( game ).not_to be_nil
+      expect( game ).to be_kind_of Hash
+    end
+
+    [ :code, :starts_at, :teams, :division, :category ].each do |attribute|
+      it "should extract the games #{attribute}" do
+        expect( game[ attribute ] ).to eq( game_attributes[ attribute.to_s ] )
+      end
+    end
+
+  end 
+
+  shared_context "without referee" do
+
+    it "should not have a referee" do
+      expect( game.keys ).not_to include( :referee )
+    end
+
+  end
+
+  shared_context "with referee" do
+
+    it "should have a referee key" do
+      expect( game.keys ).to include( :referee )
+    end
+
+    it "should extract the correct referee" do
+      expect( game[ :referee ] ).to eq( game_attributes[ 'referee' ] )
+    end
+
+  end
+
+  shared_context "without location" do
+
+    it "should not have a location" do
+      expect( game.keys ).not_to include( :location )
+    end
+
+  end
+
+  shared_context "with location" do
+
+    it "should have a location attribute" do
+      expect( game.keys ).to include( :location )
+    end
+
+    it "should extract the location name" do
+      expect( game[ :location ].first ).to eq( game_attributes[ 'location' ].first )
+    end
+
+    it "should extract the location url" do
+      expect( game[ :location ].last.to_s ).to eq( game_attributes[ 'location' ].last.to_s )
+    end
+
+  end
+
+  shared_context "in the future" do
+
+    it "should not have a score" do
+      expect( game.keys ).not_to include( :scores )
+    end
+
+    it "should not have a forfait" do
+      expect( game.keys ).not_to include( :forfait )
+    end
+
+  end
+
+  shared_context "already played" do
+
+    it "should have a score" do
+      expect( game.keys ).to include( :scores )
+    end
+
+    it "should extract the score" do
+      expect( game[ :scores ] ).to eq( game_attributes[ 'scores' ] )
+    end
+
+  end
+
+  shared_context "with forfait" do
+
+    it "should have key forfait" do
+      expect( game.keys ).to include( :forfait )
+    end
+
+    it "should have the forfait set to true" do
+      expect( game[ :forfait ] ).to be_true
+    end
+
+  end
+
+  shared_context "without forfait" do
+
+    it "should not have key forfait" do
+      expect( game.keys ).not_to include( :forfait )
+    end
+
+  end
+
+
+  describe "page containing game" do
+
+    let( :page_uri ) { load_game( code ).first }
+    let( :game_attributes ) { load_game( code ).last }
+    let( :page ) { stub_korfzone_fetch( page_uri ); Korfzone::Scraper::Page.new( page_uri ) }
+    let( :game ) { page.games.find { |game| game[ :code ] == code } }
+
+    describe "V011001" do
+      let( :code ) { 'V011001' }
+
+      it_behaves_like "successful scraping"
+      it_behaves_like "basic game"
+      it_behaves_like "in the future"
+      it_behaves_like "without referee"
+      it_behaves_like "with location"
+    end
+
+    describe "V011002" do
+      let( :code ) { 'V011002' }
+
+      it_behaves_like "successful scraping"
+      it_behaves_like "basic game"
+      it_behaves_like "in the future"
+      it_behaves_like "with referee"
+      it_behaves_like "with location"
+    end
+
+    describe "V011003" do
+      let( :code ) { 'V011003' }
+
+      it_behaves_like "successful scraping"
+      it_behaves_like "basic game"
+      it_behaves_like "in the future"
+      it_behaves_like "without referee"
+      it_behaves_like "without location"
+    end
+
+    describe "V011005" do
+      let( :code ) { 'V011005' }
+
+      it_behaves_like "successful scraping"
+      it_behaves_like "basic game"
+      it_behaves_like "already played"
+      it_behaves_like "without forfait"
+      it_behaves_like "without referee"
+      it_behaves_like "with location"
+    end
+
+    describe "V011006" do
+      let( :code ) { 'V011006' }
+
+      it_behaves_like "successful scraping"
+      it_behaves_like "basic game"
+      it_behaves_like "already played"
+      it_behaves_like "without forfait"
+      it_behaves_like "with referee"
+      it_behaves_like "with location"
+    end
+
+    describe "V011007" do
+      let( :code ) { 'V011007' }
+
+      it_behaves_like "successful scraping"
+      it_behaves_like "basic game"
+      it_behaves_like "already played"
+      it_behaves_like "with forfait"
+      it_behaves_like "without referee"
+      it_behaves_like "with location"
+    end
+
+    describe "V011004" do
+      let( :code ) { 'V011004' }
+
+      it_behaves_like "successful scraping"
+
+      it "should not include a free match" do
+        expect( page.games ).to be_empty
+      end
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'simplecov'
+
+require 'korfzone/scraper'
+require 'support/open_mock'