reliefweb_scraper 0.4.1 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (3) hide show
  1. data/lib/reliefweb.rb +20 -4
  2. data/reliefweb_scraper.gemspec +2 -3
  3. metadata +101 -115
data/lib/reliefweb.rb CHANGED
@@ -1,15 +1,20 @@
1
1
  require 'hashie'
2
2
  require 'nokogiri'
3
3
  require 'open-uri'
4
+ require 'zlib'
4
5
 
5
6
  module Reliefweb
6
7
 
8
+ URL = "http://reliefweb.int/disasters/rss.xml?sl=environment-disaster_listing"
9
+
7
10
  class Disaster < Hashie::Mash; end
8
11
 
9
12
  def self.featured_disasters(options = {})
10
- url = 'http://reliefweb.int/disasters/rss.xml?sl=environment-disaster_listing'
11
- xml = Nokogiri::XML(open(url))
12
- xml.xpath('//channel/item').map do |item|
13
+ xml = Nokogiri::XML(rss)
14
+ items = xml.xpath('//channel/item')
15
+ raise(StandardError, "Reliefweb:featured_disasters is empty!!! Please check #{URL}") if items.empty?
16
+
17
+ items.map do |item|
13
18
  title = item.xpath('title').text.sub(/-[^-]*$/, '').strip
14
19
  url = item.xpath('link').text
15
20
  country_code = item.xpath('reliefweb:iso3').map(&:text).join(', ')
@@ -23,5 +28,16 @@ module Reliefweb
23
28
  end
24
29
  end
25
30
 
26
- end
31
+ private
32
+
33
+ #
34
+ # Get the RSS and inflate it if necessary
35
+ # >> open('http://en.wikipedia.org/wiki/France').content_encoding #=> ["gzip"]
36
+ # >> open('http://en.wikipedia.org/wiki/Thailand').content_encoding #=> []
37
+ #
38
+ def self.rss
39
+ stream = open(URL)
40
+ stream.content_encoding.empty? ? stream.read : Zlib::GzipReader.new(stream).read
41
+ end
27
42
 
43
+ end
data/reliefweb_scraper.gemspec CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "reliefweb_scraper"
6
- s.version = "0.4.1"
6
+ s.version = "0.4.2"
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.authors = ["Nathan Broadbent", "Stanley Lau", "Stephen Kenworthy"]
9
9
  s.email = ["itdept@crossroads.org.hk"]
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
16
16
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
17
  s.require_paths = ["lib"]
18
18
 
19
- s.add_dependency('nokogiri', ">= 1.5.0")
19
+ s.add_dependency('nokogiri', ">= 1.5.5")
20
20
  s.add_dependency('hashie', "1.1.0")
21
21
 
22
22
  s.add_development_dependency("rspec", "~> 2.5.0")
@@ -24,4 +24,3 @@ Gem::Specification.new do |s|
24
24
  s.add_development_dependency("webmock", "~> 1.7.0")
25
25
  s.add_development_dependency("awesome_print", "~> 0.4.0")
26
26
  end
27
-
metadata CHANGED
@@ -1,131 +1,122 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: reliefweb_scraper
3
- version: !ruby/object:Gem::Version
4
- hash: 13
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.4.2
5
5
  prerelease:
6
- segments:
7
- - 0
8
- - 4
9
- - 1
10
- version: 0.4.1
11
6
  platform: ruby
12
- authors:
7
+ authors:
13
8
  - Nathan Broadbent
14
9
  - Stanley Lau
15
10
  - Stephen Kenworthy
16
11
  autorequire:
17
12
  bindir: bin
18
13
  cert_chain: []
19
-
20
- date: 2012-03-24 00:00:00 +08:00
21
- default_executable:
22
- dependencies:
23
- - !ruby/object:Gem::Dependency
24
- requirement: &id001 !ruby/object:Gem::Requirement
25
- none: false
26
- requirements:
27
- - - ">="
28
- - !ruby/object:Gem::Version
29
- hash: 3
30
- segments:
31
- - 1
32
- - 5
33
- - 0
34
- version: 1.5.0
35
- version_requirements: *id001
14
+ date: 2012-10-25 00:00:00.000000000 Z
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
36
17
  name: nokogiri
37
- prerelease: false
18
+ requirement: !ruby/object:Gem::Requirement
19
+ none: false
20
+ requirements:
21
+ - - ! '>='
22
+ - !ruby/object:Gem::Version
23
+ version: 1.5.5
38
24
  type: :runtime
39
- - !ruby/object:Gem::Dependency
40
- requirement: &id002 !ruby/object:Gem::Requirement
25
+ prerelease: false
26
+ version_requirements: !ruby/object:Gem::Requirement
41
27
  none: false
42
- requirements:
43
- - - "="
44
- - !ruby/object:Gem::Version
45
- hash: 19
46
- segments:
47
- - 1
48
- - 1
49
- - 0
50
- version: 1.1.0
51
- version_requirements: *id002
28
+ requirements:
29
+ - - ! '>='
30
+ - !ruby/object:Gem::Version
31
+ version: 1.5.5
32
+ - !ruby/object:Gem::Dependency
52
33
  name: hashie
53
- prerelease: false
34
+ requirement: !ruby/object:Gem::Requirement
35
+ none: false
36
+ requirements:
37
+ - - '='
38
+ - !ruby/object:Gem::Version
39
+ version: 1.1.0
54
40
  type: :runtime
55
- - !ruby/object:Gem::Dependency
56
- requirement: &id003 !ruby/object:Gem::Requirement
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ none: false
44
+ requirements:
45
+ - - '='
46
+ - !ruby/object:Gem::Version
47
+ version: 1.1.0
48
+ - !ruby/object:Gem::Dependency
49
+ name: rspec
50
+ requirement: !ruby/object:Gem::Requirement
57
51
  none: false
58
- requirements:
52
+ requirements:
59
53
  - - ~>
60
- - !ruby/object:Gem::Version
61
- hash: 27
62
- segments:
63
- - 2
64
- - 5
65
- - 0
54
+ - !ruby/object:Gem::Version
66
55
  version: 2.5.0
67
- version_requirements: *id003
68
- name: rspec
69
- prerelease: false
70
56
  type: :development
71
- - !ruby/object:Gem::Dependency
72
- requirement: &id004 !ruby/object:Gem::Requirement
57
+ prerelease: false
58
+ version_requirements: !ruby/object:Gem::Requirement
73
59
  none: false
74
- requirements:
60
+ requirements:
75
61
  - - ~>
76
- - !ruby/object:Gem::Version
77
- hash: 57
78
- segments:
79
- - 1
80
- - 11
81
- - 1
82
- version: 1.11.1
83
- version_requirements: *id004
62
+ - !ruby/object:Gem::Version
63
+ version: 2.5.0
64
+ - !ruby/object:Gem::Dependency
84
65
  name: vcr
85
- prerelease: false
66
+ requirement: !ruby/object:Gem::Requirement
67
+ none: false
68
+ requirements:
69
+ - - ~>
70
+ - !ruby/object:Gem::Version
71
+ version: 1.11.1
86
72
  type: :development
87
- - !ruby/object:Gem::Dependency
88
- requirement: &id005 !ruby/object:Gem::Requirement
73
+ prerelease: false
74
+ version_requirements: !ruby/object:Gem::Requirement
89
75
  none: false
90
- requirements:
76
+ requirements:
91
77
  - - ~>
92
- - !ruby/object:Gem::Version
93
- hash: 11
94
- segments:
95
- - 1
96
- - 7
97
- - 0
98
- version: 1.7.0
99
- version_requirements: *id005
78
+ - !ruby/object:Gem::Version
79
+ version: 1.11.1
80
+ - !ruby/object:Gem::Dependency
100
81
  name: webmock
101
- prerelease: false
82
+ requirement: !ruby/object:Gem::Requirement
83
+ none: false
84
+ requirements:
85
+ - - ~>
86
+ - !ruby/object:Gem::Version
87
+ version: 1.7.0
102
88
  type: :development
103
- - !ruby/object:Gem::Dependency
104
- requirement: &id006 !ruby/object:Gem::Requirement
89
+ prerelease: false
90
+ version_requirements: !ruby/object:Gem::Requirement
105
91
  none: false
106
- requirements:
92
+ requirements:
107
93
  - - ~>
108
- - !ruby/object:Gem::Version
109
- hash: 15
110
- segments:
111
- - 0
112
- - 4
113
- - 0
114
- version: 0.4.0
115
- version_requirements: *id006
94
+ - !ruby/object:Gem::Version
95
+ version: 1.7.0
96
+ - !ruby/object:Gem::Dependency
116
97
  name: awesome_print
117
- prerelease: false
98
+ requirement: !ruby/object:Gem::Requirement
99
+ none: false
100
+ requirements:
101
+ - - ~>
102
+ - !ruby/object:Gem::Version
103
+ version: 0.4.0
118
104
  type: :development
119
- description: Gathers disasters from Reliefweb via RSS and presents them as a Disaster hashie
120
- email:
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ none: false
108
+ requirements:
109
+ - - ~>
110
+ - !ruby/object:Gem::Version
111
+ version: 0.4.0
112
+ description: Gathers disasters from Reliefweb via RSS and presents them as a Disaster
113
+ hashie
114
+ email:
121
115
  - itdept@crossroads.org.hk
122
116
  executables: []
123
-
124
117
  extensions: []
125
-
126
118
  extra_rdoc_files: []
127
-
128
- files:
119
+ files:
129
120
  - .gitignore
130
121
  - .rspec
131
122
  - Gemfile
@@ -136,39 +127,34 @@ files:
136
127
  - spec/fixtures/vcr_cassettes/reliefweb.yml
137
128
  - spec/lib/reliefweb_spec.rb
138
129
  - spec/spec_helper.rb
139
- has_rdoc: true
140
130
  homepage: http://www.crossroads.org.hk
141
131
  licenses: []
142
-
143
132
  post_install_message:
144
133
  rdoc_options: []
145
-
146
- require_paths:
134
+ require_paths:
147
135
  - lib
148
- required_ruby_version: !ruby/object:Gem::Requirement
136
+ required_ruby_version: !ruby/object:Gem::Requirement
149
137
  none: false
150
- requirements:
151
- - - ">="
152
- - !ruby/object:Gem::Version
153
- hash: 3
154
- segments:
138
+ requirements:
139
+ - - ! '>='
140
+ - !ruby/object:Gem::Version
141
+ version: '0'
142
+ segments:
155
143
  - 0
156
- version: "0"
157
- required_rubygems_version: !ruby/object:Gem::Requirement
144
+ hash: 3681366782808176887
145
+ required_rubygems_version: !ruby/object:Gem::Requirement
158
146
  none: false
159
- requirements:
160
- - - ">="
161
- - !ruby/object:Gem::Version
162
- hash: 3
163
- segments:
147
+ requirements:
148
+ - - ! '>='
149
+ - !ruby/object:Gem::Version
150
+ version: '0'
151
+ segments:
164
152
  - 0
165
- version: "0"
153
+ hash: 3681366782808176887
166
154
  requirements: []
167
-
168
155
  rubyforge_project:
169
- rubygems_version: 1.6.2
156
+ rubygems_version: 1.8.21
170
157
  signing_key:
171
158
  specification_version: 3
172
159
  summary: Feeds disasters from Reliefweb
173
160
  test_files: []
174
-