course_scraper 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rvmrc

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional.
+environment_id="1.9.3@course_scraper"
+
+#
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficicent compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+#
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments" \
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+
+  if [[ -s ".rvm/hooks/after_use" ]]
+  then
+    . ".rvm/hooks/after_use"
+  fi
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  if ! rvm --create  "$environment_id"
+  then
+    echo "Failed to create RVM environment ''."
+  fi
+fi
+
+#
+# If you use an RVM gemset file to install a list of gems (*.gems), you can have
+# it be automatically loaded. Uncomment the following and adjust the filename if
+# necessary.
+#
+# filename=".gems"
+# if [[ -s "$filename" ]] ; then
+#   rvm gemset import "$filename" | grep -v already | grep -v listed | grep -v complete | sed '/^$/d'
+# fi
+
+#
+# If you use bundler and would like to run bundle each time you enter the
+# directory, you can uncomment the following code.
+#
+export PATH="./bin:$PATH"
+#

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in course_scraper.gemspec
+gemspec
+gem 'rake'

data/Rakefile

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['./test/**/*_test.rb']
+end
+
+task :default => [:test]

data/Readme.md

@@ -0,0 +1,41 @@
+# course_scraper
+
+A course scraper that gets all the vocational training courses in Catalonia
+and Spain.
+
+# Install
+
+In your Gemfile:
+
+    gem 'course_scraper', git: 'git://github.com/codegram/course_scraper'
+
+## Usage
+
+```ruby
+require 'course_scraper'
+catalan_categories = CourseScraper::Catalonia.scrape
+
+catalan_categories.each do |category|
+  category.name
+  # => "Administracio"
+  category.courses.each do |course|
+    course.name
+    # => "Transport"
+    course.type
+    # => :medium
+  end
+end
+
+spanish_categories = CourseScraper::Spain.scrape
+
+spanish_categories.each do |category|
+  category.name
+  # => "Administracion"
+  category.courses.each do |course|
+    course.name
+    # => "Transporte"
+    course.type
+    # => :medium
+  end
+end
+```

data/course_scraper.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "course_scraper/version"
+
+Gem::Specification.new do |s|
+  s.name        = "course_scraper"
+  s.version     = CourseScraper::VERSION
+  s.authors     = ["Josep M. Bach"]
+  s.email       = ["josep.m.bach@gmail.com"]
+  s.homepage    = "http://github.com/codegram/course_scraper"
+  s.summary     = %q{Gives you a list of all courses in Spanish vocational education.}
+  s.description = %q{Gives you a list of all courses in Spanish vocational education.}
+
+  s.rubyforge_project = "course_scraper"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "capybara"
+  s.add_runtime_dependency "capybara-webkit"
+  s.add_development_dependency "vcr"
+  s.add_development_dependency "webmock"
+  s.add_development_dependency "mocha"
+  s.add_development_dependency "minitest"
+end

data/lib/course_scraper.rb

@@ -0,0 +1,9 @@
+require_relative 'course_scraper/version'
+require_relative 'course_scraper/course'
+require_relative 'course_scraper/category'
+require_relative 'course_scraper/catalonia'
+require_relative 'course_scraper/spain'
+
+module CourseScraper
+end
+

data/lib/course_scraper/catalonia.rb

@@ -0,0 +1,107 @@
+require 'capybara'
+require 'capybara/dsl'
+require 'capybara-webkit'
+
+module CourseScraper
+  # Public: A scraper for all the vocational training courses in Catalonia.
+  #
+  # Examples
+  #
+  #   courses = Catalonia.scrape
+  #   # => [#<CourseScraper::Course ...>, #<CourseScraper::Course ...>]
+  #
+  class Catalonia
+    include Capybara::DSL
+
+    # Public: Instantiates a new scraper and fires it to grab all the vocational
+    # training courses in Catalonia.
+    #
+    # Returns the Array collection of CourseScraper::Course instances.
+    def self.scrape
+      new.scrape
+    end
+
+    # Internal: Sets the configuration for capybara to work with the Gencat
+    # website.
+    #
+    # Returns nothing.
+    def setup_capybara
+      Capybara.run_server = false
+      Capybara.current_driver = :webkit
+      Capybara.app_host       = 'http://www20.gencat.cat'
+    end
+
+    # Public: Scrapes the vocational training courses in Catalonia.
+    #
+    # Returns the Array collection of CourseScraper::Category instances with
+    # nested Courses.
+    def scrape
+      setup_capybara
+
+      categories = []
+      each_category do |name, href|
+        categories << { name: name, url: href }
+      end
+
+      categories.map do |category|
+        cat = Category.new(category[:name], [])
+        each_course category[:url] do |name, type|
+          cat.courses << Course.new(name, type)
+        end
+        cat
+      end
+    end
+
+    # Internal: Visits the main page where the course categories are listed.
+    #
+    # Returns nothing.
+    def visit_category_list
+      visit "/portal/site/queestudiar/menuitem.d7cfc336363a7af8e85c7273b0c0e1a0/?vgnextoid=0a8137a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextchannel=0a8137a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextfmt=default"
+    end
+
+    # Internal: Call a block for every category.
+    #
+    # Yields the String name of the category and its String URL.
+    #
+    # Returns nothing.
+    def each_category(&block)
+      visit_category_list
+
+      links = []
+
+      within ".FW_bColEsquerraCos" do
+        links += all('a')
+      end
+
+      within ".FW_bColDretaCos" do
+        links += all('a')
+      end
+
+      links.each do |link|
+        block.call link.text, link[:href]
+      end
+    end
+
+    # Internal: Call a block for every course in a category URL.
+    #
+    # category_url - the String category URL
+    #
+    # Yields the String name of the course and its Symbol type.
+    #
+    # Returns nothing.
+    def each_course(category_url, &block)
+      visit category_url
+
+      courses = []
+
+      all('.CEDU_vertical').each do |ul|
+        type = ul.find('li:nth-of-type(2)').text =~ /superior/ ? :high : :medium
+        courses << [ul.find('li.titol').text, type]
+      end
+
+      courses.each do |course|
+        block.call *course
+      end
+    end
+  end
+end

data/lib/course_scraper/category.rb

@@ -0,0 +1,16 @@
+module CourseScraper
+  # Public: A Category of vocational courses that contains them.
+  #
+  class Category
+    attr_reader :name, :courses
+
+    # Public: Initializes a new Category.
+    #
+    # name    - the String name of the category
+    # courses - the Array of courses of the category
+    def initialize(name, courses)
+      @name    = name
+      @courses = courses || []
+    end
+  end
+end

data/lib/course_scraper/course.rb

@@ -0,0 +1,16 @@
+module CourseScraper
+  # Public: A vocational course.
+  #
+  class Course
+    attr_reader :name, :type
+
+    # Public: Initializes a new Course.
+    #
+    # name - the String name of the course
+    # type - the Symbol type of the course (either :high or :medium).
+    def initialize(name, type)
+      @name = name
+      @type = type
+    end
+  end
+end

data/lib/course_scraper/spain.rb

@@ -0,0 +1,107 @@
+# encoding: utf-8
+require 'capybara'
+require 'capybara/dsl'
+require 'capybara-webkit'
+
+module CourseScraper
+  # Public: A scraper for all the vocational training courses in Spain.
+  #
+  # Examples
+  #
+  #   courses = Spain.scrape
+  #   # => [#<CourseScraper::Course ...>, #<CourseScraper::Course ...>]
+  #
+  class Spain
+    include Capybara::DSL
+
+    # Public: Instantiates a new scraper and fires it to grab all the vocational
+    # training courses in Spain.
+    #
+    # Returns the Array collection of CourseScraper::Course instances.
+    def self.scrape
+      new.scrape
+    end
+
+    # Internal: Sets the configuration for capybara to work with the Todofp
+    # website.
+    #
+    # Returns nothing.
+    def setup_capybara
+      Capybara.run_server = false
+      Capybara.current_driver = :webkit
+      Capybara.app_host       = 'http://todofp.es'
+    end
+
+    # Public: Scrapes the vocational training courses in Spain.
+    #
+    # Returns the Array collection of CourseScraper::Category instances with
+    # nested Courses.
+    def scrape
+      setup_capybara
+
+      categories = []
+      each_category do |name, href|
+        categories << { name: name, url: href }
+      end
+
+      categories.map do |category|
+        cat = Category.new(category[:name], [])
+        each_course category[:url] do |name, type|
+          cat.courses << Course.new(name, type)
+        end
+        cat
+      end
+    end
+
+    # Internal: Visits the main page where the course categories are listed.
+    #
+    # Returns nothing.
+    def visit_category_list
+      visit '/todofp/formacion/que-y-como-estudiar/oferta-formativa/todos-los-estudios.html'
+    end
+
+    # Internal: Call a block for every category.
+    #
+    # Yields the String name of the category and its String URL.
+    #
+    # Returns nothing.
+    def each_category(&block)
+      visit_category_list
+
+      links = []
+
+      within ".columnas-fp" do
+        links = all('a')
+      end
+
+      links.each do |link|
+        block.call link.text, link[:href]
+      end
+    end
+
+    # Internal: Call a block for every course in a category URL.
+    #
+    # category_url - the String category URL
+    #
+    # Yields the String name of the course and its Symbol type.
+    #
+    # Returns nothing.
+    def each_course(category_url, &block)
+      visit category_url
+
+      courses = []
+
+      all('.columnas-familiafp ul:nth-of-type(1) li').each do |course|
+        courses << [course.text, :medium]
+      end
+
+      all('.columnas-familiafp ul:nth-of-type(2) li').each do |course|
+        courses << [course.text, :high]
+      end
+
+      courses.each do |course|
+        block.call *course
+      end
+    end
+  end
+end

data/lib/course_scraper/version.rb

@@ -0,0 +1,3 @@
+module CourseScraper
+  VERSION = "0.0.1"
+end

data/test/course_scraper/catalonia_test.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require_relative '../test_helper'
+
+module CourseScraper
+  describe Catalonia do
+    subject { Catalonia.new }
+
+    before do
+      subject.setup_capybara
+    end
+
+    describe ".scrape" do
+      it 'delegates to an instance' do
+        Catalonia.stubs(:new).returns subject
+        subject.expects(:scrape)
+
+        Catalonia.scrape
+      end
+    end
+
+    describe "#visit_category_list" do
+      it 'visit the category list' do
+        subject.expects(:visit)
+        subject.visit_category_list
+      end
+    end
+
+    describe "#each_category" do
+      it 'calls a block for each category' do
+        names = []
+        urls  = []
+        subject.each_category do |name, url|
+          names << name
+          urls << url
+        end
+
+        names = names[0..2]
+        urls  = urls[0..2]
+
+        names.must_equal [
+          "Activitats físiques i esportives",
+          "Agrària",
+          "Comerç i màrqueting"
+        ]
+        urls.must_equal [
+          "/portal/site/queestudiar/menuitem.796f7d19c318c94cd56a1c76b0c0e1a0/?vgnextoid=e01237a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextchannel=e01237a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextfmt=default",
+          "/portal/site/queestudiar/menuitem.796f7d19c318c94cd56a1c76b0c0e1a0/?vgnextoid=d6e237a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextchannel=d6e237a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextfmt=default",
+          "/portal/site/queestudiar/menuitem.796f7d19c318c94cd56a1c76b0c0e1a0/?vgnextoid=1d8337a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextchannel=1d8337a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextfmt=default"
+        ]
+      end
+    end
+
+    describe "#each_course" do
+      it 'calls a block for each course in a category' do
+        category_url = "http://www20.gencat.cat/portal/site/queestudiar/menuitem.796f7d19c318c94cd56a1c76b0c0e1a0/?vgnextoid=e01237a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextchannel=e01237a9f4f2b210VgnVCM2000009b0c1e0aRCRD&vgnextfmt=default://google.com"
+
+        names = []
+        types = []
+
+        VCR.use_cassette(:catalan_courses) do
+          subject.each_course(category_url) do |name, type|
+            names << name
+            types << type
+          end
+        end
+
+        names[0].must_match /i esportives/
+        names[1].must_match /sicoesportives en el medi natural/
+        names[2].must_match /en el medi natural, perfil professional/
+
+        types[0].must_equal :high
+        types[1].must_equal :medium
+        types[2].must_equal :medium
+      end
+    end
+  end
+end

data/test/course_scraper/spain_test.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+require_relative '../test_helper'
+
+module CourseScraper
+  describe Spain do
+    subject { Spain.new }
+
+    before do
+      subject.setup_capybara
+    end
+
+    describe ".scrape" do
+      it 'delegates to an instance' do
+        Spain.stubs(:new).returns subject
+        subject.expects(:scrape)
+
+        Spain.scrape
+      end
+    end
+
+    describe "#visit_category_list" do
+      it 'visit the category list' do
+        subject.expects(:visit)
+        subject.visit_category_list
+      end
+    end
+
+    describe "#each_category" do
+      it 'calls a block for each category' do
+        names = []
+        urls  = []
+        subject.each_category do |name, url|
+          names << name
+          urls << url
+        end
+
+        names = names[0..2]
+        urls  = urls[0..2]
+
+        names.must_equal [
+          "Actividades Físicas y Deportivas",
+          "Administración y Gestión",
+          "Agraria"
+        ]
+        urls.must_equal [
+          "/todofp/formacion/que-y-como-estudiar/oferta-formativa/todos-los-estudios/actividades-fisico-deportivas.html",
+          "/todofp/formacion/que-y-como-estudiar/oferta-formativa/todos-los-estudios/administracion-gestion.html",
+          "/todofp/formacion/que-y-como-estudiar/oferta-formativa/todos-los-estudios/agraria.html"
+        ]
+      end
+    end
+
+    describe "#each_course" do
+      it 'calls a block for each course in a category' do
+        category_url = "http://todofp.es/todofp/formacion/que-y-como-estudiar/oferta-formativa/todos-los-estudios/actividades-fisico-deportivas.html"
+
+        names = []
+        types = []
+
+        VCR.use_cassette(:spanish_courses) do
+          subject.each_course(category_url) do |name, type|
+            names << name
+            types << type
+          end
+        end
+
+        names[0].must_equal "Conducción de actividades físico-deportivas en el medio natural"
+        names[1].must_equal "Animación de actividades físico-deportivas"
+
+        types[0].must_equal :medium
+        types[1].must_equal :high
+      end
+    end
+  end
+end
+

data/test/test_helper.rb

@@ -0,0 +1,12 @@
+gem 'minitest'
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'mocha'
+require 'vcr'
+require_relative '../lib/course_scraper'
+
+VCR.config do |c|
+  c.cassette_library_dir = "test/fixtures/cassettes"
+  c.stub_with :webmock
+  c.default_cassette_options = { :record => :new_episodes }
+end

metadata

@@ -0,0 +1,159 @@
+--- !ruby/object:Gem::Specification
+name: course_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Josep M. Bach
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara-webkit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gives you a list of all courses in Spanish vocational education.
+email:
+- josep.m.bach@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- Rakefile
+- Readme.md
+- course_scraper.gemspec
+- lib/course_scraper.rb
+- lib/course_scraper/catalonia.rb
+- lib/course_scraper/category.rb
+- lib/course_scraper/course.rb
+- lib/course_scraper/spain.rb
+- lib/course_scraper/version.rb
+- test/course_scraper/catalonia_test.rb
+- test/course_scraper/spain_test.rb
+- test/test_helper.rb
+homepage: http://github.com/codegram/course_scraper
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: course_scraper
+rubygems_version: 1.8.19
+signing_key: 
+specification_version: 3
+summary: Gives you a list of all courses in Spanish vocational education.
+test_files:
+- test/course_scraper/catalonia_test.rb
+- test/course_scraper/spain_test.rb
+- test/test_helper.rb