reliefweb_scraper 0.3.1 → 0.4.0

data/README.md

@@ -1,18 +1,16 @@
 Reliefweb Scraper
 =================
 
-* Scrapes featured disasters from http://reliefweb.int/home
-* Retrieves extra information by searching http://glidenumber.net
+* Feeds featured disasters from http://reliefweb.int/
 
-Objects such as Reliefweb::Disaster and Glidenumber::Record are subclasses of Hashie::Mash,
-so you can work with them in the following ways:
+Reliefweb::Disaster is a subclass of Hashie::Mash, so you can do the following:
 
     disaster = Reliefweb.featured_disasters.first
     puts disaster["title"]
     puts disaster.title
 
 
-* Run "rake featured_disasters:fetch" to see the disaster data that is scraped
+* Run "rake featured_disasters:fetch" to see the disaster data
 * Run "rake spec" to run the rspec tests
 ** Note: run "rm spec/fixtures/vcr_cassettes/*.yml" if you want to re-record the vcr specs
 
@@ -21,11 +19,7 @@ Examples
 
 Return an array of featured disasters:
 
-    Reliefweb.featured_disasters(:verbose => true)
-
-Fetch disaster information from http://glidenumber.net :
-
-    Glidenumber.find("OT-2011-000110-UGA")
+    Reliefweb.featured_disasters
 
 
 History
@@ -33,3 +27,4 @@ History
 
 * version 0.2.0 - initial release
 * version 0.3.1 - updated to reflect new changes to the ReliefWeb website
+* version 0.4.0 - using new RSS feed from ReliefWeb, no need for GlideNumber anymore

data/Rakefile

@@ -15,7 +15,7 @@ namespace :featured_disasters do
   task :fetch do
     require 'reliefweb'
     require 'awesome_print'
-    ap Reliefweb.featured_disasters(:verbose => true)
+    ap Reliefweb.featured_disasters
   end
 end
 

data/lib/reliefweb.rb

@@ -1,35 +1,27 @@
-require 'date'
 require 'hashie'
-require 'mechanize'
-require File.join(File.dirname(__FILE__), 'glidenumber')
+require 'nokogiri'
+require 'open-uri'
 
 module Reliefweb
-  class ParseError < StandardError; end
 
-  def self.agent
-    @agent ||= Mechanize.new
-  end
+  class Disaster < Hashie::Mash; end
 
   def self.featured_disasters(options = {})
-    agent.get("http://reliefweb.int/home")
-    links = agent.page.parser.css("div#middle-right.page-middle div.region div.block-content div.view div.view-content table.views-view-grid tbody tr td div.grid-item div.views-field span.field-content a")
-    raise ParseError.new("Could not find any featured disasters on Reliefweb") if links.empty?
-    links.map do |link|
-      title = link.text.sub(/-[^-]*$/, '').strip
-      href = link.attributes["href"].to_s
-      puts "== Fetching glide number from #{href} (#{title})..." if options[:verbose]
-      reliefweb_glide = fetch_glidenumber(href)
-      puts "== Fetching glide details for #{reliefweb_glide}..." if options[:verbose]
-      record = Glidenumber.find(reliefweb_glide)
-      Disaster.new({:title => title, :url => URI.join("http://reliefweb.int", href).to_s}.merge(record.to_hash))
+    url = 'http://reliefweb.int/disasters/rss.xml?sl=environment-disaster_listing'
+    xml = Nokogiri::XML(open(url))
+    xml.xpath('//channel/item').map do |item|
+      title = item.xpath('title').text.sub(/-[^-]*$/, '').strip
+      url = item.xpath('link').text
+      country_code = item.xpath('reliefweb:iso3').map(&:text).join(', ')
+      date = Time.parse(item.xpath('pubDate').text)
+      glidenumber = item.xpath('reliefweb:glide').text
+      disaster_type = item.xpath('reliefweb:disaster_type').map(&:text).join(', ')
+      current = true
+      Disaster.new({:title => title, :url => url, :country_code => country_code,
+        :date => date, :glidenumber => glidenumber, :disaster_type => disaster_type,
+        :current => current})
     end
   end
 
-  def self.fetch_glidenumber(link)
-    agent.get(link)
-    agent.page.parser.css("div.views-field-entity-id-3 div.field-content").text.strip
-  end
-
-  class Disaster < Hashie::Mash; end
 end
 

data/reliefweb_scraper.gemspec

@@ -3,21 +3,21 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "reliefweb_scraper"
-  s.version     = "0.3.1"
+  s.version     = "0.4.0"
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Nathan Broadbent", "Stanley Lau", "Stephen Kenworthy"]
   s.email       = ["itdept@crossroads.org.hk"]
   s.homepage    = "http://www.crossroads.org.hk"
-  s.summary     = %q{Scrape Reliefweb featured disasters}
-  s.description = %q{Scrapes reliefweb's featured disasters, and pulls additional information from glidenumber.net}
+  s.summary     = %q{Feeds disasters from Reliefweb}
+  s.description = %q{Gathers disasters from Reliefweb via RSS and presents them as a Disaster hashie}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency('mechanize', ">= 2.0.1")
-  s.add_dependency('hashie',    ">= 1.1.0")
+  s.add_dependency('nokogiri', ">= 1.5.0")
+  s.add_dependency('hashie',   ">= 1.1.0")
 
   s.add_development_dependency("rspec", "~> 2.5.0")
   s.add_development_dependency("vcr", "~> 1.11.1")