reliefweb_scraper 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. data/lib/reliefweb.rb +20 -4
  2. data/reliefweb_scraper.gemspec +2 -3
  3. metadata +101 -115
data/lib/reliefweb.rb CHANGED
@@ -1,15 +1,20 @@
1
1
  require 'hashie'
2
2
  require 'nokogiri'
3
3
  require 'open-uri'
4
+ require 'zlib'
4
5
 
5
6
  module Reliefweb
6
7
 
8
+ URL = "http://reliefweb.int/disasters/rss.xml?sl=environment-disaster_listing"
9
+
7
10
  class Disaster < Hashie::Mash; end
8
11
 
9
12
  def self.featured_disasters(options = {})
10
- url = 'http://reliefweb.int/disasters/rss.xml?sl=environment-disaster_listing'
11
- xml = Nokogiri::XML(open(url))
12
- xml.xpath('//channel/item').map do |item|
13
+ xml = Nokogiri::XML(rss)
14
+ items = xml.xpath('//channel/item')
15
+ raise(StandardError, "Reliefweb:featured_disasters is empty!!! Please check #{URL}") if items.empty?
16
+
17
+ items.map do |item|
13
18
  title = item.xpath('title').text.sub(/-[^-]*$/, '').strip
14
19
  url = item.xpath('link').text
15
20
  country_code = item.xpath('reliefweb:iso3').map(&:text).join(', ')
@@ -23,5 +28,16 @@ module Reliefweb
23
28
  end
24
29
  end
25
30
 
26
- end
31
+ private
32
+
33
+ #
34
+ # Get the RSS and inflate it if necessary
35
+ # >> open('http://en.wikipedia.org/wiki/France').content_encoding #=> ["gzip"]
36
+ # >> open('http://en.wikipedia.org/wiki/Thailand').content_encoding #=> []
37
+ #
38
+ def self.rss
39
+ stream = open(URL)
40
+ stream.content_encoding.empty? ? stream.read : Zlib::GzipReader.new(stream).read
41
+ end
27
42
 
43
+ end
data/reliefweb_scraper.gemspec CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "reliefweb_scraper"
6
- s.version = "0.4.1"
6
+ s.version = "0.4.2"
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.authors = ["Nathan Broadbent", "Stanley Lau", "Stephen Kenworthy"]
9
9
  s.email = ["itdept@crossroads.org.hk"]
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
16
16
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
17
  s.require_paths = ["lib"]
18
18
 
19
- s.add_dependency('nokogiri', ">= 1.5.0")
19
+ s.add_dependency('nokogiri', ">= 1.5.5")
20
20
  s.add_dependency('hashie', "1.1.0")
21
21
 
22
22
  s.add_development_dependency("rspec", "~> 2.5.0")
@@ -24,4 +24,3 @@ Gem::Specification.new do |s|
24
24
  s.add_development_dependency("webmock", "~> 1.7.0")
25
25
  s.add_development_dependency("awesome_print", "~> 0.4.0")
26
26
  end
27
-
metadata CHANGED
@@ -1,131 +1,122 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: reliefweb_scraper
3
- version: !ruby/object:Gem::Version
4
- hash: 13
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.4.2
5
5
  prerelease:
6
- segments:
7
- - 0
8
- - 4
9
- - 1
10
- version: 0.4.1
11
6
  platform: ruby
12
- authors:
7
+ authors:
13
8
  - Nathan Broadbent
14
9
  - Stanley Lau
15
10
  - Stephen Kenworthy
16
11
  autorequire:
17
12
  bindir: bin
18
13
  cert_chain: []
19
-
20
- date: 2012-03-24 00:00:00 +08:00
21
- default_executable:
22
- dependencies:
23
- - !ruby/object:Gem::Dependency
24
- requirement: &id001 !ruby/object:Gem::Requirement
25
- none: false
26
- requirements:
27
- - - ">="
28
- - !ruby/object:Gem::Version
29
- hash: 3
30
- segments:
31
- - 1
32
- - 5
33
- - 0
34
- version: 1.5.0
35
- version_requirements: *id001
14
+ date: 2012-10-25 00:00:00.000000000 Z
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
36
17
  name: nokogiri
37
- prerelease: false
18
+ requirement: !ruby/object:Gem::Requirement
19
+ none: false
20
+ requirements:
21
+ - - ! '>='
22
+ - !ruby/object:Gem::Version
23
+ version: 1.5.5
38
24
  type: :runtime
39
- - !ruby/object:Gem::Dependency
40
- requirement: &id002 !ruby/object:Gem::Requirement
25
+ prerelease: false
26
+ version_requirements: !ruby/object:Gem::Requirement
41
27
  none: false
42
- requirements:
43
- - - "="
44
- - !ruby/object:Gem::Version
45
- hash: 19
46
- segments:
47
- - 1
48
- - 1
49
- - 0
50
- version: 1.1.0
51
- version_requirements: *id002
28
+ requirements:
29
+ - - ! '>='
30
+ - !ruby/object:Gem::Version
31
+ version: 1.5.5
32
+ - !ruby/object:Gem::Dependency
52
33
  name: hashie
53
- prerelease: false
34
+ requirement: !ruby/object:Gem::Requirement
35
+ none: false
36
+ requirements:
37
+ - - '='
38
+ - !ruby/object:Gem::Version
39
+ version: 1.1.0
54
40
  type: :runtime
55
- - !ruby/object:Gem::Dependency
56
- requirement: &id003 !ruby/object:Gem::Requirement
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ none: false
44
+ requirements:
45
+ - - '='
46
+ - !ruby/object:Gem::Version
47
+ version: 1.1.0
48
+ - !ruby/object:Gem::Dependency
49
+ name: rspec
50
+ requirement: !ruby/object:Gem::Requirement
57
51
  none: false
58
- requirements:
52
+ requirements:
59
53
  - - ~>
60
- - !ruby/object:Gem::Version
61
- hash: 27
62
- segments:
63
- - 2
64
- - 5
65
- - 0
54
+ - !ruby/object:Gem::Version
66
55
  version: 2.5.0
67
- version_requirements: *id003
68
- name: rspec
69
- prerelease: false
70
56
  type: :development
71
- - !ruby/object:Gem::Dependency
72
- requirement: &id004 !ruby/object:Gem::Requirement
57
+ prerelease: false
58
+ version_requirements: !ruby/object:Gem::Requirement
73
59
  none: false
74
- requirements:
60
+ requirements:
75
61
  - - ~>
76
- - !ruby/object:Gem::Version
77
- hash: 57
78
- segments:
79
- - 1
80
- - 11
81
- - 1
82
- version: 1.11.1
83
- version_requirements: *id004
62
+ - !ruby/object:Gem::Version
63
+ version: 2.5.0
64
+ - !ruby/object:Gem::Dependency
84
65
  name: vcr
85
- prerelease: false
66
+ requirement: !ruby/object:Gem::Requirement
67
+ none: false
68
+ requirements:
69
+ - - ~>
70
+ - !ruby/object:Gem::Version
71
+ version: 1.11.1
86
72
  type: :development
87
- - !ruby/object:Gem::Dependency
88
- requirement: &id005 !ruby/object:Gem::Requirement
73
+ prerelease: false
74
+ version_requirements: !ruby/object:Gem::Requirement
89
75
  none: false
90
- requirements:
76
+ requirements:
91
77
  - - ~>
92
- - !ruby/object:Gem::Version
93
- hash: 11
94
- segments:
95
- - 1
96
- - 7
97
- - 0
98
- version: 1.7.0
99
- version_requirements: *id005
78
+ - !ruby/object:Gem::Version
79
+ version: 1.11.1
80
+ - !ruby/object:Gem::Dependency
100
81
  name: webmock
101
- prerelease: false
82
+ requirement: !ruby/object:Gem::Requirement
83
+ none: false
84
+ requirements:
85
+ - - ~>
86
+ - !ruby/object:Gem::Version
87
+ version: 1.7.0
102
88
  type: :development
103
- - !ruby/object:Gem::Dependency
104
- requirement: &id006 !ruby/object:Gem::Requirement
89
+ prerelease: false
90
+ version_requirements: !ruby/object:Gem::Requirement
105
91
  none: false
106
- requirements:
92
+ requirements:
107
93
  - - ~>
108
- - !ruby/object:Gem::Version
109
- hash: 15
110
- segments:
111
- - 0
112
- - 4
113
- - 0
114
- version: 0.4.0
115
- version_requirements: *id006
94
+ - !ruby/object:Gem::Version
95
+ version: 1.7.0
96
+ - !ruby/object:Gem::Dependency
116
97
  name: awesome_print
117
- prerelease: false
98
+ requirement: !ruby/object:Gem::Requirement
99
+ none: false
100
+ requirements:
101
+ - - ~>
102
+ - !ruby/object:Gem::Version
103
+ version: 0.4.0
118
104
  type: :development
119
- description: Gathers disasters from Reliefweb via RSS and presents them as a Disaster hashie
120
- email:
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ none: false
108
+ requirements:
109
+ - - ~>
110
+ - !ruby/object:Gem::Version
111
+ version: 0.4.0
112
+ description: Gathers disasters from Reliefweb via RSS and presents them as a Disaster
113
+ hashie
114
+ email:
121
115
  - itdept@crossroads.org.hk
122
116
  executables: []
123
-
124
117
  extensions: []
125
-
126
118
  extra_rdoc_files: []
127
-
128
- files:
119
+ files:
129
120
  - .gitignore
130
121
  - .rspec
131
122
  - Gemfile
@@ -136,39 +127,34 @@ files:
136
127
  - spec/fixtures/vcr_cassettes/reliefweb.yml
137
128
  - spec/lib/reliefweb_spec.rb
138
129
  - spec/spec_helper.rb
139
- has_rdoc: true
140
130
  homepage: http://www.crossroads.org.hk
141
131
  licenses: []
142
-
143
132
  post_install_message:
144
133
  rdoc_options: []
145
-
146
- require_paths:
134
+ require_paths:
147
135
  - lib
148
- required_ruby_version: !ruby/object:Gem::Requirement
136
+ required_ruby_version: !ruby/object:Gem::Requirement
149
137
  none: false
150
- requirements:
151
- - - ">="
152
- - !ruby/object:Gem::Version
153
- hash: 3
154
- segments:
138
+ requirements:
139
+ - - ! '>='
140
+ - !ruby/object:Gem::Version
141
+ version: '0'
142
+ segments:
155
143
  - 0
156
- version: "0"
157
- required_rubygems_version: !ruby/object:Gem::Requirement
144
+ hash: 3681366782808176887
145
+ required_rubygems_version: !ruby/object:Gem::Requirement
158
146
  none: false
159
- requirements:
160
- - - ">="
161
- - !ruby/object:Gem::Version
162
- hash: 3
163
- segments:
147
+ requirements:
148
+ - - ! '>='
149
+ - !ruby/object:Gem::Version
150
+ version: '0'
151
+ segments:
164
152
  - 0
165
- version: "0"
153
+ hash: 3681366782808176887
166
154
  requirements: []
167
-
168
155
  rubyforge_project:
169
- rubygems_version: 1.6.2
156
+ rubygems_version: 1.8.21
170
157
  signing_key:
171
158
  specification_version: 3
172
159
  summary: Feeds disasters from Reliefweb
173
160
  test_files: []
174
-