congress-scrapper 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (17) hide show
  1. data/.gitignore +4 -0
  2. data/Gemfile +4 -0
  3. data/README.md +50 -0
  4. data/Rakefile +1 -0
  5. data/congress-scrapper.gemspec +27 -0
  6. data/lib/congress-scrapper/version.rb +5 -0
  7. data/lib/congress-scrapper.rb +79 -0
  8. data/spec/fixtures/closed_proposal_page.html +788 -0
  9. data/spec/fixtures/open_proposal_page.html +651 -0
  10. data/spec/fixtures/proposers.yml +9 -0
  11. data/spec/fixtures/search_page.html +881 -0
  12. data/spec/fixtures/search_results_page1.html +526 -0
  13. data/spec/fixtures/search_results_page2.html +531 -0
  14. data/spec/lib/scrapper_spec.rb +63 -0
  15. data/spec/lib/scrapper_spec_helper.rb +41 -0
  16. data/spec/spec_helper.rb +7 -0
  17. metadata +116 -0
data/spec/lib/scrapper_spec_helper.rb ADDED
@@ -0,0 +1,41 @@
1
+ require File.dirname(__FILE__) + "/../spec_helper"
2
+
3
+ module CongressWebSitePaths
4
+ def search_page
5
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada"
6
+ end
7
+
8
+ def search_results_page
9
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/enviarCgiBuscadorAvIniciativas"
10
+ end
11
+
12
+ def search_results_next_page
13
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&FMT=INITXLTS.fmt&DOCS=26-50&DOCORDER=FIFO&OPDEF=Y&QUERY=%40FECH%26gt%3B%3D20091101+%26+%40FECH%26lt%3B%3D20100417+%26+%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR."
14
+ end
15
+
16
+ def proposal_page1
17
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&PIECE=IWA9&FMT=INITXD1S.fmt&FORM1=INITXLTS.fmt&DOCS=1-1&QUERY=%40FECH%26gt%3B%3D20091101+%26+%40FECH%26lt%3B%3D20100417+%26+%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR."
18
+ end
19
+
20
+ def proposal_page2
21
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&PIECE=IWA9&FMT=INITXD1S.fmt&FORM1=INITXLTS.fmt&DOCS=2-2&QUERY=%40FECH%26gt%3B%3D20091101+%26+%40FECH%26lt%3B%3D20100417+%26+%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR."
22
+ end
23
+
24
+ def proposal_page3
25
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&PIECE=IWA9&FMT=INITXD1S.fmt&FORM1=INITXLTS.fmt&DOCS=26-26&QUERY=%40FECH%26gt%3B%3D20091101+%26+%40FECH%26lt%3B%3D20100417+%26+%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR."
26
+ end
27
+
28
+ def proposal_page4
29
+ "http://www.congreso.es/portal/page/portal/Congreso/Congreso/Iniciativas/Busqueda%20Avanzada?_piref73_1335465_73_1335464_1335464.next_page=/wc/servidorCGI&CMD=VERLST&BASE=IWI9&PIECE=IWA9&FMT=INITXD1S.fmt&FORM1=INITXLTS.fmt&DOCS=2-2&QUERY=%28I%29.ACIN1.+%26+%28%22COMPETENCIA+LEGISLATIVA+PLENA%22%29.TPTR.+%26+%28%22APROBADO+SIN+MODIFICACIONES%22%29.CIER."
30
+ end
31
+
32
+ end
33
+
34
+ module HelperMethods
35
+ def fixture(name)
36
+ File.read(File.dirname(__FILE__) + "/../fixtures/#{name}.html")
37
+ end
38
+ end
39
+
40
+ RSpec.configuration.include HelperMethods
41
+ RSpec.configuration.include CongressWebSitePaths
data/spec/spec_helper.rb ADDED
@@ -0,0 +1,7 @@
1
+ # This file is copied to spec/ when you run 'rails generate rspec:install'
2
+ require 'rspec'
3
+ require 'webmock/rspec'
4
+
5
+ RSpec.configure do |config|
6
+ config.include WebMock::API
7
+ end
metadata ADDED
@@ -0,0 +1,116 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: congress-scrapper
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Luismi Cavallé
9
+ - Raimond García
10
+ - Alberto Fernández-Capel
11
+ autorequire:
12
+ bindir: bin
13
+ cert_chain: []
14
+ date: 2011-10-22 00:00:00.000000000Z
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: rspec
18
+ requirement: &2152233780 !ruby/object:Gem::Requirement
19
+ none: false
20
+ requirements:
21
+ - - ! '>='
22
+ - !ruby/object:Gem::Version
23
+ version: '0'
24
+ type: :development
25
+ prerelease: false
26
+ version_requirements: *2152233780
27
+ - !ruby/object:Gem::Dependency
28
+ name: webmock
29
+ requirement: &2152233100 !ruby/object:Gem::Requirement
30
+ none: false
31
+ requirements:
32
+ - - ! '>='
33
+ - !ruby/object:Gem::Version
34
+ version: '0'
35
+ type: :development
36
+ prerelease: false
37
+ version_requirements: *2152233100
38
+ - !ruby/object:Gem::Dependency
39
+ name: progressbar
40
+ requirement: &2152232560 !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ! '>='
44
+ - !ruby/object:Gem::Version
45
+ version: '0'
46
+ type: :runtime
47
+ prerelease: false
48
+ version_requirements: *2152232560
49
+ - !ruby/object:Gem::Dependency
50
+ name: mechanize
51
+ requirement: &2152232140 !ruby/object:Gem::Requirement
52
+ none: false
53
+ requirements:
54
+ - - ! '>='
55
+ - !ruby/object:Gem::Version
56
+ version: '0'
57
+ type: :runtime
58
+ prerelease: false
59
+ version_requirements: *2152232140
60
+ description: Scrapper to get proposals from Spanish Congress
61
+ email:
62
+ - voodoorai2000 at gmail
63
+ executables: []
64
+ extensions: []
65
+ extra_rdoc_files: []
66
+ files:
67
+ - .gitignore
68
+ - Gemfile
69
+ - README.md
70
+ - Rakefile
71
+ - congress-scrapper.gemspec
72
+ - lib/congress-scrapper.rb
73
+ - lib/congress-scrapper/version.rb
74
+ - spec/fixtures/closed_proposal_page.html
75
+ - spec/fixtures/open_proposal_page.html
76
+ - spec/fixtures/proposers.yml
77
+ - spec/fixtures/search_page.html
78
+ - spec/fixtures/search_results_page1.html
79
+ - spec/fixtures/search_results_page2.html
80
+ - spec/lib/scrapper_spec.rb
81
+ - spec/lib/scrapper_spec_helper.rb
82
+ - spec/spec_helper.rb
83
+ homepage: http://github.com/agoraciudadana/congress-scrapper
84
+ licenses: []
85
+ post_install_message:
86
+ rdoc_options: []
87
+ require_paths:
88
+ - lib
89
+ required_ruby_version: !ruby/object:Gem::Requirement
90
+ none: false
91
+ requirements:
92
+ - - ! '>='
93
+ - !ruby/object:Gem::Version
94
+ version: '0'
95
+ required_rubygems_version: !ruby/object:Gem::Requirement
96
+ none: false
97
+ requirements:
98
+ - - ! '>='
99
+ - !ruby/object:Gem::Version
100
+ version: '0'
101
+ requirements: []
102
+ rubyforge_project: congress-scrapper
103
+ rubygems_version: 1.8.10
104
+ signing_key:
105
+ specification_version: 3
106
+ summary: Scrapper to get proposals from Spanish Congress
107
+ test_files:
108
+ - spec/fixtures/closed_proposal_page.html
109
+ - spec/fixtures/open_proposal_page.html
110
+ - spec/fixtures/proposers.yml
111
+ - spec/fixtures/search_page.html
112
+ - spec/fixtures/search_results_page1.html
113
+ - spec/fixtures/search_results_page2.html
114
+ - spec/lib/scrapper_spec.rb
115
+ - spec/lib/scrapper_spec_helper.rb
116
+ - spec/spec_helper.rb