semantic-crawler 0.2.0 → 0.3.0

data/lib/semantic_crawler/linked_geo_data/relevant_nodes.rb

@@ -10,15 +10,19 @@ module SemanticCrawler
                 "rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
             }
 
+            # @attribute [r]
             # The input latitude value
             attr_reader :latitude
 
+            # @attribute [r]
             # The input longitude value
             attr_reader :longitude
 
+            # @attribute [r]
             # The input radius value in meters
             attr_reader :radius
 
+            # @attribute [r]
             # The linkedgeodata.org link to the relevant nodes
             attr_reader :url
 
@@ -41,6 +45,7 @@ module SemanticCrawler
 
             # Returns an array of SemanticCrawler::LinkedGeoData::RelevantNode
             # objects
+            # @return [Array<SemanticCrawler::LinkedGeoData::RelevantNode>]
             def relevant_nodes
                 nodeset = query_root_node("rdf:Description", @@NAMESPACES)
                 @items = []
@@ -61,6 +66,7 @@ module SemanticCrawler
             end
 
             # Outputs the document as XML
+            # @return [String] The document serialized as XML
             def xml_document
                 @root_node.to_s
             end

data/lib/semantic_crawler/version.rb

@@ -1,5 +1,5 @@
 module SemanticCrawler
 
     # The current version of this library.
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
 end

data/semantic_crawler.gemspec

@@ -0,0 +1,35 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+# Maintain your gem's version:
+require "semantic_crawler/version"
+
+# Describe your gem and declare its dependencies:
+Gem::Specification.new do |s|
+    s.name        = "semantic-crawler"
+    s.version     = SemanticCrawler::VERSION
+    s.authors     = ["Alex Oberhauser"]
+    s.email       = ["alex.oberhauser@sigimera.org"]
+    s.homepage    = "https://github.com/obale/semantic_crawler"
+    s.summary     = "SemanticCrawler is a ruby library that encapsulates data gathering from different sources."
+    s.description = "SemanticCrawler is a ruby library that encapsulates data gathering from different sources. Currently country information from Factbook and FAO (Food and Agriculture Organization of the United Nations), crisis information from GDACS.org and geo data from LinkedGeoData are supported. Additional the GeoNames module allows to get Factbook and FAO country information from GPS coordinates."
+
+    #s.files = Dir["{app,config,db,lib,log}/**/*"] + ["MIT-LICENSE", "Rakefile", "README.rdoc"]
+    s.files = `git ls-files`.split("\n")
+    s.executables = `git ls-files -- bin/*`.split('\n').map{ |f| File.basename(f) }
+    s.test_files = Dir["{test,spec}/**/*"]
+    s.require_paths = ['lib']
+
+    s.add_dependency "httparty"
+    s.add_dependency "json"
+    #s.add_dependency "google-api-client" # Freebase API access
+    s.add_dependency "nokogiri"           # XML Parsing
+    s.add_dependency "geonames"           # Use for the GeoNames module
+
+    s.add_development_dependency "yard"
+    s.add_development_dependency "rails", "~> 3.2.2"
+    s.add_development_dependency "sqlite3"
+    s.add_development_dependency "rspec-rails"
+    s.add_development_dependency "simplecov"
+    s.add_development_dependency "simplecov-rcov"
+
+end

data/spec/dbpedia_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe SemanticCrawler::Dbpedia do
+    pending "Implement 'Dbpedia' module"
+
+    before(:all) do
+    end
+
+end

data/spec/factbook_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+
+describe SemanticCrawler::Factbook do
+
+    before(:all) do
+        @austria = SemanticCrawler::Factbook::Country.new("Austria")
+    end
+
+    it "test nil country" do
+        wrongCountry = SemanticCrawler::Factbook::Country.new(nil)
+        wrongCountry.url.nil?.should == true
+    end
+
+    it "test unknown country" do
+        wrongCountry = SemanticCrawler::Factbook::Country.new("xyz")
+        wrongCountry.landboundary.nil?.should == true
+        wrongCountry.background.nil?.should == true
+        wrongCountry.name.nil?.should == true
+    end
+
+    it "init austria country information" do
+        be_valid @austria.country_name.eql?("Austria")
+        be_valid @austria.url.eql?("http://www4.wiwiss.fu-berlin.de/factbook/data/austria")
+    end
+
+    it "get doc dump" do
+        be_valid !@austria.xml_document.empty?
+    end
+
+    it "get austria name" do
+        value = @austria.name
+        be_valid value.to_s.eql?("Austria")
+    end
+
+    it "get austria background information" do
+        value = @austria.background
+        be_valid !value.nil?
+    end
+
+    it "get austria population information" do
+        value = @austria.population_total
+        be_valid !value.nil?
+    end
+
+    it "get austria legalsystem property (not wrapped)" do
+        value = @austria.get_factbook_property("legalsystem")
+        be_valid !value.nil?
+    end
+
+    it "get austria latitude" do
+        value = @austria.latitude
+        be_valid value.to_s.eql?("47.3333333333333")
+    end
+
+    it "get austria longitude" do
+        value = @austria.longitude
+        be_valid value.to_s.eql?("13.3333333333333")
+    end
+
+    it "get austria landboundary" do
+        value = @austria.landboundary
+        be_valid value.count == 7
+    end
+
+    it "get austria terrain" do
+        value = @austria.terrain
+        be_valid !value.nil?
+    end
+
+    it "get austria total airports" do
+        value = @austria.airports_total
+        be_valid value.to_s.eql?("55")
+    end
+
+    it "get austria heliports" do
+        value = @austria.heliports
+        be_valid value.to_s.eql?("1")
+    end
+
+    it "get austria climate" do
+        value = @austria.climate
+        be_valid !value.nil?
+    end
+
+    it "get austria location description" do
+        value = @austria.location
+        be_valid !value.nil?
+    end
+end

data/spec/fao_austria_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+
+describe SemanticCrawler::Fao do
+
+    before(:all) do
+        @obj = SemanticCrawler::Fao::Country.new("Austria")
+    end
+
+    it "init empty country object" do
+        @obj = SemanticCrawler::Fao::Country.new(nil)
+        @obj.country_name.should.eql?(nil)
+        @obj.url.should eq(nil)
+    end
+
+    it "check austria object" do
+        @obj.country_name.should.eql?("Austria")
+        @obj.url.should eq("http://www.fao.org/countryprofiles/geoinfo/geopolitical/data/Austria")
+    end
+
+    it "get austria RDF/XML dump" do
+        be_valid !@obj.xml_document.empty?
+    end
+
+    it "get austria type" do
+        @obj.type_url.should eq("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/self_governing")
+    end
+
+    it "get austria location (max/min lat, max/min long)" do
+        @obj.max_latitude.should eq("49.02")
+        @obj.max_longitude.should eq("17.16")
+        @obj.min_latitude.should eq("46.38")
+        @obj.min_longitude.should eq("9.54")
+    end
+
+    it "get land area" do
+        @obj.land_area_notes.should eq("Official data reported on FAO Questionnaires from countries")
+        @obj.land_area_total.should eq("8243.5")
+        @obj.land_area_unit.should eq("1000 Ha")
+        @obj.land_area_year.should eq("2009")
+    end
+
+    it "get currency in en/es/fr" do
+        @obj.name_currency("en").should eq("euro")
+        @obj.name_currency("es").should eq("euro")
+        @obj.name_currency("fr").should eq("euro")
+    end
+
+    it "get official name in en/es/fr" do
+        @obj.official_name("en").should eq("the Republic of Austria")
+        @obj.official_name("es").should eq("la República de Austria")
+        @obj.official_name("fr").should eq("la République d'Autriche")
+    end
+
+    it "get is in group (URL)" do
+        group = @obj.is_in_group_url
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/EU").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Europe").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/FAO").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/World").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/western_Europe").should be_true
+    end
+
+    it "get is in group (Name)" do
+        group = @obj.is_in_group_name
+        group.include?("EU").should be_true
+        group.include?("Europe").should be_true
+        group.include?("FAO").should be_true
+        group.include?("World").should be_true
+        group.include?("western_Europe").should be_true
+    end
+
+    it "get has boarder with (URL)" do
+        group = @obj.has_boarder_with_url
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Czech_Republic_the").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Germany").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Hungary").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Italy").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Liechtenstein").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Slovakia").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Slovenia").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Switzerland").should be_true
+    end
+
+    it "get has boarder with (Name)" do
+        group = @obj.has_boarder_with_name
+        group.include?("Czech_Republic_the").should be_true
+        group.include?("Germany").should be_true
+        group.include?("Hungary").should be_true
+        group.include?("Italy").should be_true
+        group.include?("Liechtenstein").should be_true
+        group.include?("Slovakia").should be_true
+        group.include?("Slovenia").should be_true
+        group.include?("Switzerland").should be_true
+    end
+
+    it "get same as links" do
+        links = @obj.same_as
+        links.include?("http://aims.fao.org/aos/agrovoc/c_718").should be_true
+        links.include?("http://dbpedia.org/resource/Austria").should be_true
+    end
+
+    it "get population information (notes/total/unit/year)" do
+        @obj.population_notes.should eq("2010 Revision from the UN Population Division")
+        @obj.population_total.should eq("8394.0")
+        @obj.population_unit.should eq("1000")
+        @obj.population_year.should eq("2010")
+    end
+
+    it "get valid since/until" do
+        @obj.valid_since.should eq("1985")
+        @obj.valid_until.should eq("9999")
+    end
+
+    it "get dbpedia id" do
+        @obj.code_dbpedia_id.should eq("Austria")
+    end
+
+end

data/spec/fao_papua_new_guinea_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe SemanticCrawler::Fao do
+
+    before(:all) do
+        @obj = SemanticCrawler::Fao::Country.new("Papua New Guinea")
+    end
+
+    it "check papua new guinea object" do
+        @obj.country_name.should eq("Papua New Guinea")
+        @obj.url.should eq("http://www.fao.org/countryprofiles/geoinfo/geopolitical/data/Papua_New_Guinea")
+    end
+
+    it "get papua new guinea RDF/XML dump" do
+        be_valid !@obj.xml_document.empty?
+    end
+
+    it "get papa new guinea type" do
+        @obj.type_url.should eq("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/self_governing")
+    end
+
+    it "get papa new guinea location (max/min lat, max/min long)" do
+        @obj.max_latitude.should eq("-1.13")
+        @obj.max_longitude.should eq("159.49")
+        @obj.min_latitude.should eq("-11.66")
+        @obj.min_longitude.should eq("140.87")
+    end
+
+    it "get land area" do
+        @obj.land_area_notes.should eq("Manual Estimation")
+        @obj.land_area_total.should eq("45286.0")
+        @obj.land_area_unit.should eq("1000 Ha")
+        @obj.land_area_year.should eq("2009")
+    end
+
+    it "get currency in en/es/fr" do
+        @obj.name_currency("en").should eq("kina")
+        @obj.name_currency("es").should eq("kina")
+        @obj.name_currency("fr").should eq("kina")
+    end
+
+    it "get official name in en/es/fr" do
+        @obj.official_name("en").should eq("Papua New Guinea")
+        @obj.official_name("es").should eq("Papua Nueva Guinea")
+        @obj.official_name("fr").should eq("la Papouasie-Nouvelle-Guinée")
+    end
+
+    it "get is in group (URL)" do
+        group = @obj.is_in_group_url
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Oceania").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Melanesia").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/FAO").should be_true
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/World").should be_true
+    end
+
+    it "get is in group (Name)" do
+        group = @obj.is_in_group_name
+        group.include?("Oceania").should be_true
+        group.include?("Melanesia").should be_true
+        group.include?("FAO").should be_true
+        group.include?("World").should be_true
+    end
+
+    it "get has boarder with (URL)" do
+        group = @obj.has_boarder_with_url
+        group.include?("http://www.fao.org/countryprofiles/geoinfo/geopolitical/resource/Indonesia").should be_true
+    end
+
+    it "get has boarder with (Name)" do
+        group = @obj.has_boarder_with_name
+        group.include?("Indonesia").should be_true
+    end
+
+    it "get same as links" do
+        links = @obj.same_as
+        links.include?("http://aims.fao.org/aos/agrovoc/c_5555").should be_true
+        links.include?("http://dbpedia.org/resource/Papua_New_Guinea").should be_true
+    end
+
+    it "get population information (notes/total/unit/year)" do
+        @obj.population_notes.should eq("2010 Revision from the UN Population Division")
+        @obj.population_total.should eq("6858.0")
+        @obj.population_unit.should eq("1000")
+        @obj.population_year.should eq("2010")
+    end
+
+    it "get valid since/until" do
+        @obj.valid_since.should eq("1985")
+        @obj.valid_until.should eq("9999")
+    end
+
+    it "get dbpedia id" do
+        @obj.code_dbpedia_id.should eq("Papua_New_Guinea")
+    end
+
+end
+

data/spec/freebase_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe SemanticCrawler::Freebase do
+
+    it "test empty country" do
+        emptyCountry = SemanticCrawler::Freebase::Country.new(nil)
+        emptyCountry.get.nil?.should == true
+    end
+
+    it "init austria country information" do
+        austria = SemanticCrawler::Freebase::Country.new("Austria")
+        austria.get.to_s.nil?.should == false
+    end
+
+    pending "Implement the 'Freebase' module"
+
+end

data/spec/gdacs_spec.rb

@@ -0,0 +1,111 @@
+require 'spec_helper'
+
+describe SemanticCrawler::Gdacs do
+    before(:all) do
+        @feed = SemanticCrawler::Gdacs::Feed.new
+    end
+
+    it "test wrong url" do
+        noFeed = SemanticCrawler::Gdacs::Feed.new(nil)
+        be_valid noFeed.nil?
+    end
+
+    it "init gdacs feed" do
+        be_valid !@feed.url.to_s.empty?
+    end
+
+    it "get channel title" do
+        be_valid !@feed.title.to_s.empty?
+    end
+
+    it "get channel description" do
+        be_valid !@feed.description.to_s.empty?
+    end
+
+    it "get channel link" do
+        be_valid !@feed.link.to_s.empty?
+    end
+
+    it "get feed dump" do
+        be_valid !@feed.xml_document.empty?
+    end
+
+    it "get channel pubDate" do
+        be_valid !@feed.pubDate.nil?
+    end
+
+    it "get channel webMaster" do
+        be_valid @feed.webMaster.to_s.include?("@")
+    end
+
+    it "get channel managingEditor" do
+        be_valid @feed.managingEditor.to_s.include?("@")
+    end
+
+    it "get channel atom:link" do
+        be_valid @feed.atom_link.to_s.start_with?("http")
+    end
+
+    it "get channel items + resources" do
+        items = @feed.items
+        be_valid !items.nil?
+        items.each do |item|
+            be_valid !item.title.to_s.empty?
+            be_valid !item.description.nil?
+            be_valid !item.link.to_s.empty?
+            be_valid !item.pubDate.nil?
+            be_valid !item.fromdate.nil?
+            be_valid !item.todate.nil?
+            be_valid !item.subject.nil?
+            be_valid !item.guid.nil?
+            be_valid !item.latitude.to_s.empty?
+            be_valid !item.longitude.to_s.empty?
+            be_valid !item.version.nil?
+            be_valid !item.eventtype.nil?
+            be_valid !item.alertlevel.nil?
+            be_valid !item.eventname.nil?
+            be_valid !item.eventid.nil?
+            be_valid !item.episodeid.nil?
+            be_valid !item.severity.nil?
+            be_valid !item.population.nil?
+            be_valid !item.vulnerability.nil?
+            be_valid !item.country.nil?
+            be_valid !item.iso3.nil?
+            be_valid !item.glide.nil?
+            be_valid !item.enclosure_url.nil?
+            be_valid !item.enclosure_type.nil?
+            be_valid !item.enclosure_length.nil?
+            resources = item.resources
+            resources.each do |resource|
+                be_valid !resource.title.to_s.empty?
+                be_valid !resource.url.to_s.empty?
+                be_valid !resource.type.to_s.empty?
+                be_valid !resource.description.nil?
+                be_valid !resource.accesslevel.nil?
+                be_valid !resource.xslt.nil?
+                be_valid !resource.acknowledgements.nil?
+            end
+        end
+    end
+
+    it "test emergency feed" do
+        noFeed = SemanticCrawler::Gdacs::EmergencyFeed.new(nil)
+        be_valid noFeed.nil?
+
+        @emergency_feed = SemanticCrawler::Gdacs::EmergencyFeed.new
+        be_valid !@emergency_feed.xml_document.empty?
+        @emergency_feed.title.to_s.should_not be_empty
+        @emergency_feed.description.to_s.should_not be_empty
+        items = @emergency_feed.items
+        items.each do |item|
+            item.link.to_s.should_not be_empty
+            item.title.to_s.should_not be_empty
+            item.eventType.to_s.should_not be_empty
+            item.pubDate.to_s.should_not be_empty
+            item.lastUpdate.to_s.should_not be_empty
+            item.glide.to_s.should_not be_empty
+        end
+
+    end
+
+end