congress-scrapper 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in congress-scrapper.gemspec
+gemspec

data/README.md

@@ -0,0 +1,50 @@
+# Congress Scrapper
+
+## Introduction
+
+Congress Scrapper is a Ruby gem to scrape the official Spanish parliament webpage.
+
+## Usage
+
+```ruby
+  proposals = Congress::Scrapper.scrape # Array with Congress proposals
+  proposal = proposals.first
+  
+  puts proposal.proposal_type
+  >> "Proyecto de Ley"
+  
+  puts proposal.title
+  >> "Proyecto de Ley de almacenamiento geológico de dióxido de carbono"
+```
+
+## Install
+
+<pre>
+  gem install congress-scrapper
+</pre>
+
+## Contribute
+
+1. Find or create an issue
+    
+2. Add a comment to the issue to let people know you're going to work on it
+    
+3. Fork
+    
+4. Hack your changes in a topic branch (don't forget to write some tests ;)
+    
+5. Make pull request
+    
+6. Wait for comments from maintainers or code merge
+
+
+## Authors
+
+Original author: Luismi Cavallé
+
+Code extracted as a ruby gem by: Raimond García and Alberto Fernández-Capel
+
+
+## License
+
+Released under the MIT license.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/congress-scrapper.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "congress-scrapper/version"
+
+Gem::Specification.new do |s|
+  s.name        = "congress-scrapper"
+  s.version     = Congress::Scrapper::VERSION
+  s.authors     = ["Luismi Cavallé", "Raimond García", "Alberto Fernández-Capel"]
+  s.email       = ["voodoorai2000 at gmail"]
+  s.homepage    = "http://github.com/agoraciudadana/congress-scrapper"
+  s.summary     = %q{Scrapper to get proposals from Spanish Congress}
+  s.description = %q{Scrapper to get proposals from Spanish Congress}
+
+  s.rubyforge_project = "congress-scrapper"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "webmock"
+  
+  s.add_runtime_dependency "progressbar"
+  s.add_runtime_dependency "mechanize"
+end

data/lib/congress-scrapper/version.rb

@@ -0,0 +1,5 @@
+module Congress
+  module Scrapper
+    VERSION = "0.0.1"
+  end
+end

data/lib/congress-scrapper.rb

@@ -0,0 +1,79 @@
+# coding: utf-8
+
+require "congress-scrapper/version"
+require "mechanize"
+require "progressbar"
+
+module Congress
+  module Scrapper
+    extend self
+
+    def agent
+      @agent ||= Mechanize.new
+    end
+
+    def scrape
+      search_page = agent.get("http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada")
+      search_form = search_page.form_with(:action => /enviarCgiBuscadorAvIniciativas/)
+      search_form["TPTR"] = "Competencia Legislativa Plena"
+      results_page = search_form.submit
+
+      total_results = results_page.search("//*[contains(text(), 'Iniciativas encontradas')]/span").first.text.to_i
+      progress = ProgressBar.new("Scrapping", total_results)
+      
+      proposals = []
+
+      while results_page
+        results_page.search(".titulo_iniciativa a").each do |title|
+          @proposal_page = agent.get(title[:href])
+          
+          proposal_type = clean_text(text_for(".subtitulo_competencias"))
+          
+          resolution = clean_text(text_for("//*[@class='apartado_iniciativa' and contains(normalize-space(text()),'Resultado de la tramitación')]/following-sibling::*[@class='texto']"))
+
+          commission_name = clean_text(text_for("//*[@class='apartado_iniciativa' and contains(normalize-space(text()),'Comisión competente:')]/following-sibling::*[@class='texto']"))
+
+          proposer = clean_text(text_for("//*[@class='apartado_iniciativa' and contains(normalize-space(text()),'Autor:')]/following-sibling::*[@class='texto']"))
+
+          proposed_at_text = text_for("//*[@class='texto' and contains(normalize-space(text()),'Presentado el')]")
+          proposed_at = Date.new($3.to_i, $2.to_i, $1.to_i) if proposed_at_text && proposed_at_text.match(/Presentado\s+el\s+(\d\d)\/(\d\d)\/(\d\d\d\d)/)
+
+          closed_at_text = text_for("//*[@class='apartado_iniciativa' and contains(normalize-space(text()),'Tramitación seguida por la iniciativa:')]/following-sibling::*[@class='texto']")
+          closed_at = Date.new($3.to_i, $2.to_i, $1.to_i) if closed_at_text && closed_at_text.match(/Concluido\s+.+\s+desde (\d\d)\/(\d\d)\/(\d\d\d\d)/)
+          
+          proposal = {:title               => clean_text(title.content),
+                      :official_url        => "http://www.congreso.es" + title[:href],
+                      :proposal_type       => proposal_type,
+                      :closed_at           => closed_at,
+                      :official_resolution => resolution,
+                      :commission_name     => commission_name,
+                      :proposer            => proposer,
+                      :proposed_at         => proposed_at}
+ 
+          progress.inc
+          
+          proposals << proposal
+        end
+        
+        next_page =  results_page.link_with(:text => /Siguiente/)
+        results_page = next_page.nil? ? nil : next_page.click
+      end
+      
+      progress.finish
+      
+      proposals
+    end
+
+    private
+    
+    def text_for(selector)
+      element = @proposal_page.search(selector).first
+      element.nil? ? nil : element.content
+    end
+
+    def clean_text(text)
+      return unless text
+      text.gsub(/\s+/,' ').gsub(/\s*\.\s*$/, '').strip
+    end
+  end
+end