semantic-crawler 0.0.3 → 0.0.5

data/CHANGELOG.rdoc

@@ -0,0 +1,25 @@
+= Semantic Crawler CHANGELOG
+
+== 0.0.4
+
+* Adding module for parsing country information from "Food and Agriculture
+  Organization of the United Nations"
+* Adding emergency feed to Gdacs module
+* Adding basic module structure for all modules (not all implemented)
+* Adding basic rspec sructure for all modules (not all implemented)
+
+== 0.0.3
+
+* Implementing the Gdacs module
+* Improving documentation
+* Better Exception Handling
+* Minor bug fixing
+
+== 0.0.2
+
+* Factbook::Country implemented
+* First version that is released to rubygems.org for testing purpose
+
+== 0.0.1
+
+* Initial version that is not usable and not released.

data/README.rdoc

@@ -1,6 +1,4 @@
-[ATTENTION]  This library is under heavy development. Please be patient for a usable version.
-
-= SemanticCrawler
+= SemanticCrawler {<img src="https://secure.travis-ci.org/obale/semantic_crawler.png?branch=master"/>}[http://travis-ci.org/#!/obale/semantic_crawler]
 
 This project encapsulates data gathering from different sources.
 It simplifies the extension of internal data with public available
@@ -11,8 +9,9 @@ to bypass complex NLP (natural language processing).
 == Supported Sources
 
 * CIA Factbook RDF Dump, see http://www4.wiwiss.fu-berlin.de/factbook/directory/countries
+* GDACS (see http://gdacs.org)
+* FAO - Food and Agriculture Organization of the United Nations (see http://www.fao.org)
 * [Started] LinkedGeoData - LGD (see http://linkedgeodata.org)
-* [Started] GDACS (see http://gdacs.org)
 * [Started] Freebase (see http://freebase.com)
 
 === TODO
@@ -36,23 +35,60 @@ Or from source:
 
 You can add this library also as dependency in your Gemfile:
 
-    gem "semantic-cralwer"
+    gem "semantic-crawler"
 
 Or from source:
 
-    gem "semantic-crawler", :git => "git://github.com/obale/semantic_crawler.git"                   # for the master branch or
-    gem "semantic-crawler", :git => "git://github.com/obale/semantic_crawler.git", :tags => "xyz"   # for the xyz tag
+    gem "semantic-crawler", :git => "git://github.com/obale/semantic_crawler.git"                       # for the master branch or
+    gem "semantic-crawler", :git => "git://github.com/obale/semantic_crawler.git", :branch => "develop" # for the developer branch
 
 
 == Examples
 
+    >> require "semantic_crawler"
+
+=== Factbook
+
 Fetch Factbook information about Austria:
 
-    >> require "semantic_crawler"
     >> austria = SemanticCrawler::Factbook::Country.new("austria")
     >> puts austria.background
     >> puts austria.climate
 
+=== GDACS
+
+Parse crisis information feed from GDACS.org:
+
+    >> feed = SemanticCrawler::Gdacs::Feed.new
+    >> puts feed.title.to_s
+    >> puts feed.description.to_s
+    >> feed.items.each do |item|
+    >>      puts item.title.to_s
+    >>      puts item.eventtype.to_s
+    >>      item.resources.each do |resource|
+    >>          puts resource.url.to_s
+    >>      end
+    >> end
+
+Get information from the the GDACS.org emergency feed:
+
+    >> emergency_feed = SemanticCrawler::Gdacs::EmergencyFeed.new
+    >> puts emergency_feed.title.to_s
+    >> items = @emergency_feed.items
+    >> items.each do |item|
+    >>      puts item.title.to_s
+    >>      puts item.link.to_s
+    >> end
+
+=== FAO
+
+Country information from {FAO}[http://www.fao.org]:
+
+    >> austria = SemanticCrawler::Fao::Country.new("Austria")
+    >> puts austria.name_currency("en")
+    >> puts austria.official_name("es")
+
+
 == Changelog
 
 see CHANGELOG.rdoc
@@ -63,10 +99,6 @@ see CHANGELOG.rdoc
 * Ruby 1.8.7-p358 and Rails 3.2.2
 * Ruby 1.9.3-p125 and Rails 3.2.2
 
-=== Development Branch Health
-
-{<img src="https://secure.travis-ci.org/obale/semantic_crawler.png?branch=master"/>}[http://travis-ci.org/#!/obale/semantic_crawler]
-
 
 == License
 

data/Rakefile

@@ -12,6 +12,9 @@ rescue LoadError
   RDoc::Task = Rake::RDocTask
 end
 
+require 'rake'
+require 'rspec/core/rake_task'
+
 RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_dir = 'rdoc'
   rdoc.title    = 'SemanticCrawler'
@@ -26,6 +29,9 @@ Bundler::GemHelper.install_tasks
 
 require 'rake/testtask'
 
+# Import own written tasks
+Dir.glob('lib/tasks/*.rake').each { |r| import r }
+
 Rake::TestTask.new(:test) do |t|
   t.libs << 'lib'
   t.libs << 'test'
@@ -33,4 +39,6 @@ Rake::TestTask.new(:test) do |t|
   t.verbose = false
 end
 
-task :default => :test
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => [ :test, :spec ]

data/lib/semantic_crawler.rb

@@ -13,24 +13,40 @@ require "httparty"
 #
 # The existing modules are extended stepwise and additional
 # sources are added in the future.
+require 'logger'
+
+# Top module that contains the whole library. Each sub-module
+# is wrappes one source.
 module SemanticCrawler
+    $log = Logger.new(File.expand_path('../../log/semantic-crawler.log', __FILE__), 'daily')
 end
 
-# Freebase.com - module: Freebase
-require "semantic_crawler/freebase/crawler"
-require "semantic_crawler/freebase/entity"
-require "semantic_crawler/freebase/country"
-
 # CIA Factbook RDF Dump - module: Factbook
 require "semantic_crawler/factbook"
 require "semantic_crawler/factbook/country"
 
-# LinkedGeoData (http://linkedgeodata.org) - module: LinkedGeoData
-require "semantic_crawler/linked_geo_data"
-require "semantic_crawler/linked_geo_data/relevant_node"
-
-# GDACS (http://gdacs.org) - module: Gdacs
+# GDACS.org - module: Gdacs
 require "semantic_crawler/gdacs"
 require "semantic_crawler/gdacs/feed.rb"
 require "semantic_crawler/gdacs/feed_item.rb"
 require "semantic_crawler/gdacs/resource.rb"
+require "semantic_crawler/gdacs/emergency_feed.rb"
+require "semantic_crawler/gdacs/emergency_feed_item.rb"
+
+# DBPedia - module: Dbpedia
+require "semantic_crawler/dbpedia"
+
+# GeoNames.org - module: GeoNames
+require "semantic_crawler/geo_names"
+
+# LinkedGeoData.org - module: LinkedGeoData
+require "semantic_crawler/linked_geo_data"
+require "semantic_crawler/linked_geo_data/relevant_node"
+
+# Fao.org - module: Fao
+require "semantic_crawler/fao"
+require "semantic_crawler/fao/country"
+
+# Freebase.com - module: Freebase
+require "semantic_crawler/freebase/country"
+

data/lib/semantic_crawler/dbpedia.rb

@@ -0,0 +1,4 @@
+# The module Dbpedia extracts general information from the RDF
+# dependant of the Wikipedia.
+module SemanticCrawler::Dbpedia
+end

data/lib/semantic_crawler/factbook/country.rb

@@ -35,16 +35,20 @@ module SemanticCrawler
             # Argumenst:
             #   new_country_name: (String)
             def initialize(new_country_name)
-                @country_name = new_country_name
-                @url = @@URI_PREFIX + @country_name.downcase.gsub(" ", "_").gsub("usa", "united_states")
-                begin
-                    fetch_rdf
-                rescue => e
-                    puts "Not able to get country information, through exception: " + e.message
+                if !new_country_name.nil?
+                    @country_name = new_country_name
+                    @url = @@URI_PREFIX + @country_name.downcase.gsub(" ", "_").gsub("usa", "united_states")
+                    begin
+                        fetch_rdf
+                    rescue => e
+                        $log.error("Not able to get country information, through exception: #{e}")
+                    end
                 end
             end
 
             # Returns the country name (rdfs:label)
+            # XXX: If nothing was found this method returns 
+            # <?xml version="1.0"?>
             def name
                 get_rdfs_property("label", "/rdf:RDF/rdf:Description/factbook:landboundary/factbook:Country")
             end
@@ -74,7 +78,7 @@ module SemanticCrawler
                 if !@doc.nil?
                     @doc.xpath("//factbook:landboundary/rdf:Description/@rdf:about", @@NAMESPACES)
                 else
-                    @doc = Nokogiri::XML::Document.new
+                    nil
                 end
             end
 
@@ -110,7 +114,7 @@ module SemanticCrawler
                 if !@doc.nil?
                     @doc.xpath(prefix + "/factbook:" + property_name + "/text()", @@NAMESPACES)
                 else
-                    @doc = Nokogiri::XML::Document.new
+                    nil
                 end
             end
 
@@ -120,7 +124,7 @@ module SemanticCrawler
                 if !@doc.nil?
                     @doc.xpath(prefix + "/rdfs:" + property_name + "/text()", @@NAMESPACES)
                 else
-                    @doc = Nokogiri::XML::Document.new
+                    nil
                 end
             end
 

data/lib/semantic_crawler/fao.rb

@@ -0,0 +1,6 @@
+# The module Fao retrieves information
+# from the Food and Agriculture Organization of the United Nations (see
+# http://www.fao.org)
+# It gets the data as RDF.
+module SemanticCrawler::Fao
+end

data/lib/semantic_crawler/fao/country.rb

@@ -0,0 +1,198 @@
+module SemanticCrawler
+    module Fao
+        # Represents Food and Agriculture information about one country.
+        class Country
+
+            # Namespace hash
+            @@NAMESPACES = {
+                "rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+                "fao" => "http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/",
+                "owl" => "http://www.w3.org/2002/07/owl#"
+            }
+
+            # @attribute [r]
+            # The read only country name
+            attr_reader :country_name
+
+            # @attribute [r]
+            # The read only URL to the FAO resource
+            attr_reader :url
+
+            # Initialize a new Fao country object
+            def initialize(new_country_name)
+                @country_name = new_country_name
+                @url = "http://www.fao.org/countryprofiles/geoinfo/geopolitical/data/#{@country_name}"
+                @root_node = nil
+                begin
+                    fetch_rdf
+                rescue => e
+                    $log.error("Not able to get country information, through exception: #{e}")
+                end
+            end
+
+            # The dbpedia identifier (from fao:codeDBPediaID)
+            def code_dbpedia_id
+                query_root_node("fao:codeDBPediaID/text()", @@NAMESPACES).to_s
+            end
+
+            # Links to additional information (from owl:sameAs)
+            def same_as
+                returnLinks = []
+                links = query_root_node("owl:sameAs/@rdf:resource", @@NAMESPACES)
+                links.each do |link|
+                    returnLinks << link.to_s
+                end
+                returnLinks
+            end
+
+            # The type as URL of this entity (from rdf:type)
+            def type_url
+                query_root_node("rdf:type/@rdf:resource", @@NAMESPACES).to_s
+            end
+
+            # The maximum latitude (from fao:hasMaxLatitude)
+            def max_latitude
+                query_root_node("fao:hasMaxLatitude/text()", @@NAMESPACES).to_s
+            end
+
+            # The maximum longitude (from fao:hasMaxLongitude)
+            def max_longitude
+                query_root_node("fao:hasMaxLongitude/text()", @@NAMESPACES).to_s
+            end
+
+            # The minimum latitude (from fao:hasMinLatitude)
+            def min_latitude
+                query_root_node("fao:hasMinLatitude/text()", @@NAMESPACES).to_s
+            end
+
+            # The minimum longitude (from fao:hasMinLongitude)
+            def min_longitude
+                query_root_node("fao:hasMinLongitude/text()", @@NAMESPACES).to_s
+            end
+
+            # Human readable description about the land area (from fao:landAreaNotes)
+            def land_area_notes
+                query_root_node("fao:landAreaNotes/text()", @@NAMESPACES).to_s
+            end
+
+            # Land area total value (from fao:landAreaTotal)
+            def land_area_total
+                query_root_node("fao:landAreaTotal/text()", @@NAMESPACES).to_s
+            end
+
+            # Land area unit (from fao:landAreaUnit)
+            def land_area_unit
+                query_root_node("fao:landAreaUnit/text()", @@NAMESPACES).to_s
+            end
+
+            # Land area year (from fao:landAreaYear)
+            def land_area_year
+                query_root_node("fao:landAreaYear/text()", @@NAMESPACES).to_s
+            end
+
+            # The currency name.
+            # @param [Lang] The language in which the currency name should be returned
+            def name_currency(lang)
+                query_root_node("fao:nameCurrency[@xml:lang='#{lang}']/text()", @@NAMESPACES).to_s
+            end
+
+            # The official country name
+            # @param [Lang] The language in which the official name should be returned
+            def official_name(lang)
+                query_root_node("fao:nameOfficial[@xml:lang='#{lang}']/text()", @@NAMESPACES).to_s
+            end
+
+            # Classification of this country as name (from fao:isInGroup)
+            def is_in_group_name
+                returnGroup = []
+                group = query_root_node("fao:isInGroup/@rdf:resource", @@NAMESPACES)
+                group.each do |entry|
+                    returnGroup << entry.to_s.split("/")[7]
+                end
+                returnGroup
+            end
+
+            # Classification of this country as dereferenceable URL (from
+            # fao:isInGroup)
+            def is_in_group_url
+                returnGroup = []
+                group = query_root_node("fao:isInGroup/@rdf:resource", @@NAMESPACES)
+                group.each do |entry|
+                    returnGroup << entry.to_s
+                end
+                returnGroup
+            end
+
+            # Returns all countries that share a boarder with this country (as
+            # dereferencable URL - from fao:hasBorderWith)
+            def has_boarder_with_url
+                returnGroup = []
+                group = query_root_node("fao:hasBorderWith/@rdf:resource", @@NAMESPACES)
+                group.each do |entry|
+                    returnGroup << entry.to_s
+                end
+                returnGroup
+            end
+
+            # Returns all countries that share a boarder with this country (as
+            # name)
+            def has_boarder_with_name
+                returnGroup = []
+                group = query_root_node("fao:hasBorderWith/@rdf:resource", @@NAMESPACES)
+                group.each do |entry|
+                    returnGroup << entry.to_s.split("/")[7]
+                end
+                returnGroup
+            end
+
+            # Population notes (from fao:populationNotes)
+            def population_notes
+                query_root_node("fao:populationNotes/text()", @@NAMESPACES).to_s
+            end
+
+            # Population total (from fao:populationTotal)
+            def population_total
+                query_root_node("fao:populationTotal/text()", @@NAMESPACES).to_s
+            end
+
+            # Population unit (from fao:populationUnit)
+            def population_unit
+                query_root_node("fao:populationUnit/text()", @@NAMESPACES).to_s
+            end
+
+            # Population year (from fao:populationYear)
+            def population_year
+                query_root_node("fao:populationYear/text()", @@NAMESPACES).to_s
+            end
+
+            # Entity is valid since (from fao:validSince)
+            def valid_since
+                query_root_node("fao:validSince/text()", @@NAMESPACES).to_s
+            end
+
+            # Entity is valid until (from fao:validUntil)
+            def valid_until
+                query_root_node("fao:validUntil/text()", @@NAMESPACES).to_s
+            end
+
+            # Executes a xpath query with optional a hash with namespaces
+            def query_root_node(xpath_query, namespaces = {})
+                if !@root_node.nil?
+                    @root_node.xpath(xpath_query, namespaces)
+                end
+            end
+
+            # Outputs the document as XML
+            def xml_document
+                @root_node.to_s
+            end
+
+            private
+                # Retrieves the RDF file
+                def fetch_rdf
+                    @doc = Nokogiri::XML(open(@url))
+                    @root_node = @doc.xpath("/rdf:RDF/rdf:Description", @@NAMESPACES)
+                end
+        end
+    end
+end