congress-scrapper 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (17) hide show
  1. data/.gitignore +4 -0
  2. data/Gemfile +4 -0
  3. data/README.md +50 -0
  4. data/Rakefile +1 -0
  5. data/congress-scrapper.gemspec +27 -0
  6. data/lib/congress-scrapper/version.rb +5 -0
  7. data/lib/congress-scrapper.rb +79 -0
  8. data/spec/fixtures/closed_proposal_page.html +788 -0
  9. data/spec/fixtures/open_proposal_page.html +651 -0
  10. data/spec/fixtures/proposers.yml +9 -0
  11. data/spec/fixtures/search_page.html +881 -0
  12. data/spec/fixtures/search_results_page1.html +526 -0
  13. data/spec/fixtures/search_results_page2.html +531 -0
  14. data/spec/lib/scrapper_spec.rb +63 -0
  15. data/spec/lib/scrapper_spec_helper.rb +41 -0
  16. data/spec/spec_helper.rb +7 -0
  17. metadata +116 -0
data/spec/lib/scrapper_spec_helper.rb ADDED
@@ -0,0 +1,41 @@
1
+ require File.dirname(__FILE__) + "/../spec_helper"
2
+
3
+ module CongressWebSitePaths
4
+ def search_page
5
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada"
6
+ end
7
+
8
+ def search_results_page
9
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/enviarCgiBuscadorAvIniciativas"
10
+ end
11
+
12
+ def search_results_next_page
13
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&FMT=INITXLTS.fmt&DOCS=26-50&DOCORDER=FIFO&OPDEF=Y&QUERY=%40FECH%26gt%3B%3D20091101+%26+%40FECH%26lt%3B%3D20100417+%26+%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR."
14
+ end
15
+
16
+ def proposal_page1
17
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&PIECE=IWA9&FMT=INITXD1S.fmt&FORM1=INITXLTS.fmt&DOCS=1-1&QUERY=%40FECH%26gt%3B%3D20091101+%26+%40FECH%26lt%3B%3D20100417+%26+%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR."
18
+ end
19
+
20
+ def proposal_page2
21
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&PIECE=IWA9&FMT=INITXD1S.fmt&FORM1=INITXLTS.fmt&DOCS=2-2&QUERY=%40FECH%26gt%3B%3D20091101+%26+%40FECH%26lt%3B%3D20100417+%26+%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR."
22
+ end
23
+
24
+ def proposal_page3
25
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&PIECE=IWA9&FMT=INITXD1S.fmt&FORM1=INITXLTS.fmt&DOCS=26-26&QUERY=%40FECH%26gt%3B%3D20091101+%26+%40FECH%26lt%3B%3D20100417+%26+%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR."
26
+ end
27
+
28
+ def proposal_page4
29
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&PIECE=IWA9&FMT=INITXD1S.fmt&FORM1=INITXLTS.fmt&DOCS=2-2&QUERY=%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR.+%26+%28%22APROBADO+SIN+MODIFICACIONES%22%29.CIER."
30
+ end
31
+
32
+ end
33
+
34
+ module HelperMethods
35
+ def fixture(name)
36
+ File.read(File.dirname(__FILE__) + "/../fixtures/#{name}.html")
37
+ end
38
+ end
39
+
40
+ RSpec.configuration.include HelperMethods
41
+ RSpec.configuration.include CongressWebSitePaths
data/spec/spec_helper.rb ADDED
@@ -0,0 +1,7 @@
1
+ # This file is copied to spec/ when you run 'rails generate rspec:install'
2
+ require 'rspec'
3
+ require 'webmock/rspec'
4
+
5
+ RSpec.configure do |config|
6
+ config.include WebMock::API
7
+ end
metadata ADDED
@@ -0,0 +1,116 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: congress-scrapper
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Luismi Cavallé
9
+ - Raimond García
10
+ - Alberto Fernández-Capel
11
+ autorequire:
12
+ bindir: bin
13
+ cert_chain: []
14
+ date: 2011-10-22 00:00:00.000000000Z
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: rspec
18
+ requirement: &2152233780 !ruby/object:Gem::Requirement
19
+ none: false
20
+ requirements:
21
+ - - ! '>='
22
+ - !ruby/object:Gem::Version
23
+ version: '0'
24
+ type: :development
25
+ prerelease: false
26
+ version_requirements: *2152233780
27
+ - !ruby/object:Gem::Dependency
28
+ name: webmock
29
+ requirement: &2152233100 !ruby/object:Gem::Requirement
30
+ none: false
31
+ requirements:
32
+ - - ! '>='
33
+ - !ruby/object:Gem::Version
34
+ version: '0'
35
+ type: :development
36
+ prerelease: false
37
+ version_requirements: *2152233100
38
+ - !ruby/object:Gem::Dependency
39
+ name: progressbar
40
+ requirement: &2152232560 !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ type: :runtime
47
+ prerelease: false
48
+ version_requirements: *2152232560
49
+ - !ruby/object:Gem::Dependency
50
+ name: mechanize
51
+ requirement: &2152232140 !ruby/object:Gem::Requirement
52
+ none: false
53
+ requirements:
54
+ - - ! '>='
55
+ - !ruby/object:Gem::Version
56
+ version: '0'
57
+ type: :runtime
58
+ prerelease: false
59
+ version_requirements: *2152232140
60
+ description: Scrapper to get proposals from Spanish Congress
61
+ email:
62
+ - voodoorai2000 at gmail
63
+ executables: []
64
+ extensions: []
65
+ extra_rdoc_files: []
66
+ files:
67
+ - .gitignore
68
+ - Gemfile
69
+ - README.md
70
+ - Rakefile
71
+ - congress-scrapper.gemspec
72
+ - lib/congress-scrapper.rb
73
+ - lib/congress-scrapper/version.rb
74
+ - spec/fixtures/closed_proposal_page.html
75
+ - spec/fixtures/open_proposal_page.html
76
+ - spec/fixtures/proposers.yml
77
+ - spec/fixtures/search_page.html
78
+ - spec/fixtures/search_results_page1.html
79
+ - spec/fixtures/search_results_page2.html
80
+ - spec/lib/scrapper_spec.rb
81
+ - spec/lib/scrapper_spec_helper.rb
82
+ - spec/spec_helper.rb
83
+ homepage: http://github.com/agoraciudadana/congress-scrapper
84
+ licenses: []
85
+ post_install_message:
86
+ rdoc_options: []
87
+ require_paths:
88
+ - lib
89
+ required_ruby_version: !ruby/object:Gem::Requirement
90
+ none: false
91
+ requirements:
92
+ - - ! '>='
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ required_rubygems_version: !ruby/object:Gem::Requirement
96
+ none: false
97
+ requirements:
98
+ - - ! '>='
99
+ - !ruby/object:Gem::Version
100
+ version: '0'
101
+ requirements: []
102
+ rubyforge_project: congress-scrapper
103
+ rubygems_version: 1.8.10
104
+ signing_key:
105
+ specification_version: 3
106
+ summary: Scrapper to get proposals from Spanish Congress
107
+ test_files:
108
+ - spec/fixtures/closed_proposal_page.html
109
+ - spec/fixtures/open_proposal_page.html
110
+ - spec/fixtures/proposers.yml
111
+ - spec/fixtures/search_page.html
112
+ - spec/fixtures/search_results_page1.html
113
+ - spec/fixtures/search_results_page2.html
114
+ - spec/lib/scrapper_spec.rb
115
+ - spec/lib/scrapper_spec_helper.rb
116
+ - spec/spec_helper.rb