congress-scrapper 0.0.1 → 0.0.2

data/lib/congress-scrapper/version.rb

@@ -1,5 +1,5 @@
 module Congress
   module Scrapper
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

data/lib/congress-scrapper.rb

@@ -3,6 +3,7 @@
 require "congress-scrapper/version"
 require "mechanize"
 require "progressbar"
+require_relative "proposer"
 
 module Congress
   module Scrapper
@@ -33,21 +34,20 @@ module Congress
 
           commission_name = clean_text(text_for("//*[@class='apartado_iniciativa' and contains(normalize-space(text()),'Comisión competente:')]/following-sibling::*[@class='texto']"))
 
-          proposer = clean_text(text_for("//*[@class='apartado_iniciativa' and contains(normalize-space(text()),'Autor:')]/following-sibling::*[@class='texto']"))
+          proposer_name = clean_text(text_for("//*[@class='apartado_iniciativa' and contains(normalize-space(text()),'Autor:')]/following-sibling::*[@class='texto']"))
 
           proposed_at_text = text_for("//*[@class='texto' and contains(normalize-space(text()),'Presentado el')]")
           proposed_at = Date.new($3.to_i, $2.to_i, $1.to_i) if proposed_at_text && proposed_at_text.match(/Presentado\s+el\s+(\d\d)\/(\d\d)\/(\d\d\d\d)/)
 
           closed_at_text = text_for("//*[@class='apartado_iniciativa' and contains(normalize-space(text()),'Tramitación seguida por la iniciativa:')]/following-sibling::*[@class='texto']")
           closed_at = Date.new($3.to_i, $2.to_i, $1.to_i) if closed_at_text && closed_at_text.match(/Concluido\s+.+\s+desde (\d\d)\/(\d\d)\/(\d\d\d\d)/)
-          
           proposal = {:title               => clean_text(title.content),
                       :official_url        => "http://www.congreso.es" + title[:href],
                       :proposal_type       => proposal_type,
                       :closed_at           => closed_at,
                       :official_resolution => resolution,
-                      :commission_name     => commission_name,
-                      :proposer            => proposer,
+                      :category_name       => category(commission_name),
+                      :proposer_name       => proposer(proposer_name),
                       :proposed_at         => proposed_at}
  
           progress.inc
@@ -75,5 +75,19 @@ module Congress
       return unless text
       text.gsub(/\s+/,' ').gsub(/\s*\.\s*$/, '').strip
     end
+
+    def category(name)
+      return unless name
+      upcase_first(name.gsub(/Comisión( Mixta)?( del?| para las?)? /, ""))
+    end
+
+    def upcase_first(string)
+      string[0..0].upcase + string[1..-1]
+    end
+
+    def proposer(string)
+      return unless string
+      Proposer.new(string).name
+    end
   end
-end
+end

data/lib/proposer.rb

@@ -0,0 +1,49 @@
+# coding: utf-8
+class Proposer
+  
+  attr_accessor :name
+
+  def initialize(string)
+    @name = string
+  end
+
+  def name
+    full_name ? short_name(full_name) : @name  
+  end
+
+  def full_name 
+    mapping.map(&:last).index(@name)
+  end
+
+  def short_name(index)
+    mapping[index].first
+  end
+
+  def mapping
+    [["PSOE",                "Grupo Parlamentario Socialista"],
+     ["PP",                  "Grupo Parlamentario Popular en el Congreso"],
+     ["Convergència i Unió", "Grupo Parlamentario Catalán (Convergència i Unió)"],
+     ["PNV",                 "Grupo Parlamentario Vasco (EAJ-PNV)"],
+     ["Izquierda Unida",     "Grupo Parlamentario de Esquerra Republicana-Izquierda Unida-Iniciativa per Catalunya Verds"],
+     ["Grupo Mixto",         "Grupo Parlamentario Mixto"],
+     ["PSOE",                "Senado Grupo Parlamentario Socialista"],
+     ["PP",                  "Senado Grupo Parlamentario Popular en el Senado"],
+     ["Convergència i Unió", "Senado Grupo Parlamentario Catalán en el Senado de Convergencia i Unió"],
+     ["PNV",                 "Senado Grupo Parlamentario de Senadores Nacionalistas"],
+     ["Izquierda Unida",     "Senado Grupo Parlamentario de Entesa Catalana de Progrés"],
+     ["Grupo Mixto",         "Senado Grupo Parlamentario Mixto"],
+     ["Andalucía",           "Comunidad Autónoma de Andalucía-Parlamento"],
+     ["Aragón",              "Comunidad Autónoma de Aragón-Cortes"],
+     ["Canarias",            "Comunidad Autónoma de Canarias - Parlamento"],
+     ["Castilla y León",     "Comunidad Autónoma de Castilla y León - Cortes"],
+     ["Castilla-La Mancha",  "Comunidad Autónoma de Castilla-La Mancha - Cortes"],
+     ["Cataluña",            "Comunidad Autónoma de Cataluña - Parlamento"],
+     ["Extremadura",         "Comunidad Autónoma de Extremadura - Asamblea"],
+     ["Galicia",             "Comunidad Autónoma de Galicia - Parlamento"],
+     ["Murcia",              "Comunidad Autónoma de la Región de Murcia - Asamblea Regional"],
+     ["La Rioja",            "Comunidad Autónoma de La Rioja - Diputación General"],
+     ["Baleares",            "Comunidad Autónoma de las Illes Balears - Gobierno"],
+     ["País Vasco",          "Comunidad Autónoma del País Vasco - Gobierno"],
+     ["País Vasco",          "Comunidad Autónoma del País Vasco - Parlamento"]]
+  end
+end

data/spec/lib/scrapper_spec.rb

@@ -45,8 +45,8 @@ describe Congress::Scrapper do
       proposal[:closed_at].should be_nil
       proposal[:official_resolution].should be_nil
       proposal[:proposed_at].should == Date.new(2010, 4, 9)
-      proposal[:commission_name].should == "Comisión de Medio Ambiente, Agricultura y Pesca"
-      proposal[:proposer].should == "Grupo Parlamentario Socialista"
+      proposal[:category_name].should == "Medio Ambiente, Agricultura y Pesca"
+      proposal[:proposer_name].should == "PSOE"
     end
     
     it "should populate closed proposals info" do
@@ -56,8 +56,8 @@ describe Congress::Scrapper do
       proposal[:proposal_type].should       == "Proyecto de ley"
       proposal[:closed_at].should           == Date.new(2009, 6, 24)
       proposal[:official_resolution].should == "Aprobado sin modificaciones"
-      proposal[:commission_name].should     == "Comisión de Economía y Hacienda"
-      proposal[:proposer].should            == "Gobierno"
+      proposal[:category_name].should       == "Economía y Hacienda"
+      proposal[:proposer_name].should       == "Gobierno"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: congress-scrapper
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-10-22 00:00:00.000000000Z
+date: 2012-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152233780 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -23,10 +23,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152233780
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
-  requirement: &2152233100 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -34,10 +39,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152233100
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: progressbar
-  requirement: &2152232560 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -45,10 +55,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2152232560
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &2152232140 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -56,7 +71,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2152232140
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Scrapper to get proposals from Spanish Congress
 email:
 - voodoorai2000 at gmail
@@ -71,6 +91,7 @@ files:
 - congress-scrapper.gemspec
 - lib/congress-scrapper.rb
 - lib/congress-scrapper/version.rb
+- lib/proposer.rb
 - spec/fixtures/closed_proposal_page.html
 - spec/fixtures/open_proposal_page.html
 - spec/fixtures/proposers.yml
@@ -100,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: congress-scrapper
-rubygems_version: 1.8.10
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Scrapper to get proposals from Spanish Congress