semantic-crawler 0.3.0 → 0.4.0

data/README.rdoc

@@ -1,4 +1,4 @@
-= SemanticCrawler 
+= Ruby Semantic Crawler Library
 
 This project encapsulates data gathering from different sources.
 It simplifies the extension of internal data with public available
@@ -13,7 +13,7 @@ to bypass complex NLP (natural language processing).
 * {FAO - Food and Agriculture Organization of the United Nations}[http://www.fao.org]
 * {LinkedGeoData - LGD}[http://linkedgeodata.org]
 * {GDACS}[http://gdacs.org]
-* [Started] {Freebase}[http://freebase.com]
+* {Freebase}[http://freebase.com]
 
 === TODO
 
@@ -56,13 +56,13 @@ library include or execute the following line:
 The GeoNames module is able to return a Factbook::Country and Fao::Country
 module on the base of input GPS coordinates (lat/long).
 
-    >> @innsbruck = SemanticCrawler::GeoNames::Country.new(47.271338, 11.395333)
-    >> articles = @innsbruck.get_wikipedia_articles
+    >> innsbruck = SemanticCrawler::GeoNames::Country.new(47.271338, 11.395333)
+    >> articles = innsbruck.get_wikipedia_articles
     >> articles.each do |article|
     >>      puts article.wikipedia_url
     >> end
-    >> factbook_obj = @innsbruck.get_factbook_country
-    >> fao_obj = @innsbruck.get_fao_country
+    >> factbook_obj = innsbruck.get_factbook_country
+    >> fao_obj = innsbruck.get_fao_country
 
 === Factbook
 
@@ -120,12 +120,34 @@ Geo information from {LinkedGeoData}[http://linkedgeodata.org]:
     >> end
 
 
+== Freebase
+
+Freebase.com country information:
+
+    >> country = SemanticCrawler::Freebase::Country.new("Austria")
+    >> links = country.same_as
+    >> links.each do |link|
+    >>      be_valid link.start_with?("http")
+    >> end
+    >> puts country.website
+
+
 == Tested with
 
 * Ruby 1.8.7-p358 and Rails 3.2.2
 * Ruby 1.9.3-p125 and Rails 3.2.2
 
 
+== Additional Links
+
+* {RubyGems}[http://rubygems.org/gems/semantic-crawler]
+* {RubyDoc}[http://rubydoc.info/gems/semantic-crawler/file/README.rdoc]
+* {GitHub}[https://github.com/obale/semantic_crawler]
+* {Travis CI}[http://travis-ci.org/#!/obale/semantic_crawler]
+* {Bugtracker}[https://github.com/obale/semantic_crawler/issues?sort=created&direction=desc&state=open]
+* {Wiki}[https://github.com/obale/semantic_crawler/wiki]
+
+
 == License
 
 (c) 2012 by Alex Oberhauser for {Sigimera}[http://www.sigimera.org],

data/lib/semantic_crawler/fao/country.rb

@@ -25,7 +25,7 @@ module SemanticCrawler
             def initialize(new_country_name)
                 @country_name = new_country_name
                 if !@country_name.nil?
-                    @url = @@URI_PREFIX + @country_name.gsub(" ", "_").gsub("USA", "United_States")
+                    @url = @@URI_PREFIX + @country_name.gsub(" ", "_").gsub("USA", "United_States_of_America")
                     @root_node = nil
                     begin
                         fetch_rdf

data/lib/semantic_crawler/freebase/country.rb

@@ -1,49 +1,127 @@
 # encoding: UTF-8
 
-require 'json'
-
 module SemanticCrawler
     module Freebase
         # Freebase Country entity. Currently it is very abstract and
         # could be each entry on Freebase.
-        #
-        # [XXX] The current implementation outputs only an unreadable JSON object.
         class Country
             include HTTParty
-            # The Freebase object that should be retrieved
-            attr_accessor :input_name
-            # The URL that points to the JSON object.
-            attr_accessor :json_link
+            # The URL prefix of an english Freebase RDF entity.
+            @@URI_PREFIX = "http://rdf.freebase.com/rdf/en."
+
+            # Namespace hash
+            @@NAMESPACES = {
+                "rdf" => "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+                "fb" => "http://rdf.freebase.com/ns/",
+                "owl" => "http://www.w3.org/2002/07/owl#"
+            }
 
-            # The URL prefix of an Freebase experimental JSON entity.
-            @@uri_prefix = "http://www.freebase.com/experimental/topic/standard/en/"
+            # @attribute [r]
+            # The read only country name
+            attr_reader :country_name
 
             # Creates a new Freebase object (JSON)
             def initialize(new_country_name)
-                @input_name = new_country_name
-                @html_link = nil
-                @json_link = nil
-            end
-
-            # Get Country Information from freebase.com
-            #
-            # Example:
-            #   >> austria = SemanticCrawler::Freebase::Country.new("Austria")
-            #   >> austria.get
-            #
-            # Argumenst:
-            #   options: (Hash)
-            #
-            #
-            def get(options={})
-                if !@input_name.nil?
-                    @json_link = @@uri_prefix + @input_name
-                    JSON(self.class.get(@json_link, options).body)["result"]
-                else
-                    nil
+                @country_name = new_country_name
+                if !@country_name.nil?
+                    @normalized_country = @country_name.gsub(" ", "_").gsub("USA", "United_States_of_America").downcase
+                    @url = @@URI_PREFIX + @normalized_country
+                    @subject = "http://rdf.freebase.com/ns/en.#{@normalized_country}"
+                    @root_node = nil
+                    begin
+                        fetch_rdf
+                    rescue => e
+                        $log.error("Not able to get country information, through exception: #{e}")
+                    end
                 end
             end
 
+            # Extract the fb:common.topic.topical_webpage property
+            # @return [String] Website as String
+            def website
+                query_root_node("fb:common.topic.topical_webpage/text()", @@NAMESPACES).to_s
+            end
+
+            # Extract the fb:type.object.name property
+            # @param [String] The language in ISO 3166-1 alpha-2 format
+            # @return [String] The country name
+            def object_name(lang = "en")
+                query_root_node("fb:type.object.name[@xml:lang='#{lang}']/text()", @@NAMESPACES).to_s
+            end
+
+            # Extract the fb:location.location.contains properties
+            # @return [Array<String>]
+            def contains
+                return_list = []
+                list = query_root_node("fb:location.location.contains/@rdf:resource", @@NAMESPACES)
+                if !list.nil?
+                    list.each do |entry|
+                        return_list << entry.to_s
+                    end
+                end
+                return_list
+            end
+
+            # Extract fb:location.country.administrative_divisions properties
+            # @return [Array<String>]
+            def administrative_divisions
+                return_list = []
+                list = query_root_node("fb:location.country.administrative_divisions/@rdf:resource", @@NAMESPACES)
+                if !list.nil?
+                    list.each do |entry|
+                        return_list << entry.to_s
+                    end
+                end
+                return_list
+            end
+
+            # Extract fb:government.governmental_jurisdiction.agencies
+            # @return [Array<String>] Returns the governmental jurisdiction agencies as URLs
+            def agencies
+                return_list = []
+                list = query_root_node("fb:government.governmental_jurisdiction.agencies/@rdf:resource", @@NAMESPACES)
+                if !list.nil?
+                    list.each do |entry|
+                        return_list << entry.to_s
+                    end
+                end
+                return_list
+            end
+
+            # Extract the owl:sameAs links
+            # @return [Array<String>] A list of same concept URLs, e.g. to dbpedia or nytimes
+            def same_as
+                return_list = []
+                list = query_root_node("owl:sameAs/@rdf:resource", @@NAMESPACES)
+                if !list.nil?
+                    list.each do |entry|
+                        return_list << entry.to_s
+                    end
+                end
+                return_list
+            end
+
+            # Executes a xpath query with optional a hash with namespaces
+            # @return [String]
+            def query_root_node(xpath_query, namespaces = {})
+                if !@root_node.nil?
+                    @root_node.xpath(xpath_query, namespaces)
+                end
+            end
+
+            # Outputs the document as XML
+            # @return [String] The document serialized as XML
+            def xml_document
+                @root_node.to_s
+            end
+
+            private
+            # Retrieves the RDF file
+            def fetch_rdf
+                @doc = Nokogiri::XML(open(@url))
+                @root_node = @doc.xpath("//*[@rdf:about='#{@subject}']", @@NAMESPACES)
+            end
+
         end
     end
 end

data/lib/semantic_crawler/version.rb

@@ -1,5 +1,4 @@
 module SemanticCrawler
-
     # The current version of this library.
-    VERSION = "0.3.0"
+    VERSION = "0.4.0"
 end

data/lib/semantic_crawler.rb

@@ -17,7 +17,7 @@ require "geonames"
 require 'logger'
 
 # Top module that contains the whole library. Each sub-module
-# is wrappes one source.
+# wrappes one source.
 module SemanticCrawler
     $log = Logger.new(File.expand_path('../../log/semantic-crawler.log', __FILE__), 'daily')
 end

data/lib/tasks/semantic_crawler_tasks.rake

@@ -1,4 +1,26 @@
-#desc "Explaining what the task does"
-#task :semantic_crawler do
-#  # Task goes here
-#end
+require "grit"
+
+desc "Outputs the ChangeLog of the current release (latest release on rubygems.org)"
+task :changelog do
+    repo = Grit::Repo.new('.')
+
+    currentMerge = repo.log('origin/master', nil, :merges => true)[0].id
+    lastMerge = repo.log('origin/master', nil, :merges => true)[1].id
+    changes = repo.commits_between(lastMerge, currentMerge)
+
+    tags = repo.tags
+
+    if !changes.nil?
+        changes.reverse!
+
+        tags.each do |tag|
+            if tag.commit.sha.eql?(changes[0].id)
+                puts "=> Current version: \033[0;31m#{tag.name}\033[0m (tagged)"
+            end
+        end
+
+        changes.each do |commit|
+            puts "\033[0;33m#{commit.id[0,7]}\033[0m - #{commit.date} - \033[0;32m#{commit.message}\033[0m"
+        end
+    end
+end

data/semantic_crawler.gemspec

@@ -9,11 +9,11 @@ Gem::Specification.new do |s|
     s.version     = SemanticCrawler::VERSION
     s.authors     = ["Alex Oberhauser"]
     s.email       = ["alex.oberhauser@sigimera.org"]
+    s.licenses    = ["MIT"]
     s.homepage    = "https://github.com/obale/semantic_crawler"
     s.summary     = "SemanticCrawler is a ruby library that encapsulates data gathering from different sources."
     s.description = "SemanticCrawler is a ruby library that encapsulates data gathering from different sources. Currently country information from Factbook and FAO (Food and Agriculture Organization of the United Nations), crisis information from GDACS.org and geo data from LinkedGeoData are supported. Additional the GeoNames module allows to get Factbook and FAO country information from GPS coordinates."
 
-    #s.files = Dir["{app,config,db,lib,log}/**/*"] + ["MIT-LICENSE", "Rakefile", "README.rdoc"]
     s.files = `git ls-files`.split("\n")
     s.executables = `git ls-files -- bin/*`.split('\n').map{ |f| File.basename(f) }
     s.test_files = Dir["{test,spec}/**/*"]
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
     s.add_dependency "geonames"           # Use for the GeoNames module
 
     s.add_development_dependency "yard"
+    s.add_development_dependency "grit"
     s.add_development_dependency "rails", "~> 3.2.2"
     s.add_development_dependency "sqlite3"
     s.add_development_dependency "rspec-rails"

data/spec/fao_austria_spec.rb

@@ -6,10 +6,10 @@ describe SemanticCrawler::Fao do
         @obj = SemanticCrawler::Fao::Country.new("Austria")
     end
 
-    it "init empty country object" do
-        @obj = SemanticCrawler::Fao::Country.new(nil)
-        @obj.country_name.should.eql?(nil)
-        @obj.url.should eq(nil)
+    it "test nil country" do
+        wrongCountry = SemanticCrawler::Fao::Country.new(nil)
+        wrongCountry.country_name.should.eql?(nil)
+        wrongCountry.url.should eq(nil)
     end
 
     it "check austria object" do

data/spec/freebase_spec.rb

@@ -2,16 +2,61 @@ require 'spec_helper'
 
 describe SemanticCrawler::Freebase do
 
+    before(:all) do
+        @country = SemanticCrawler::Freebase::Country.new("Austria")
+    end
+
     it "test empty country" do
         emptyCountry = SemanticCrawler::Freebase::Country.new(nil)
-        emptyCountry.get.nil?.should == true
+        emptyCountry.country_name.should_not be
+    end
+
+    it "check austria country name and xml_document" do
+        @country.country_name.should eq("Austria")
+        @country.xml_document.size.should > 0
     end
 
-    it "init austria country information" do
-        austria = SemanticCrawler::Freebase::Country.new("Austria")
-        austria.get.to_s.nil?.should == false
+    it "check same as links" do
+        links = @country.same_as
+        links.size.should > 0
+        links.each do |link|
+            be_valid link.start_with?("http")
+        end
     end
 
-    pending "Implement the 'Freebase' module"
+    it "check website" do
+        @country.website.should eq("http://www.austria.gv.at/")
+    end
+
+    it "check object name" do
+        @country.object_name.should eq("Austria")
+        @country.object_name("en").should eq("Austria")
+        @country.object_name("sl").should eq("Avstrija")
+        @country.object_name("cs").should eq("Rakousko")
+    end
+
+    it "check same as links" do
+        links = @country.contains
+        links.size.should > 0
+        links.each do |link|
+            be_valid link.start_with?("http")
+        end
+    end
+
+    it "check administrative divisions" do
+        links = @country.administrative_divisions
+        links.size.should > 0
+        links.each do |link|
+            be_valid link.start_with?("http")
+        end
+    end
+
+    it "check agencies" do
+        links = @country.agencies
+        links.size.should > 0
+        links.each do |link|
+            be_valid link.start_with?("http")
+        end
+    end
 
 end

data/test/dummy/log/development.log

@@ -53,3 +53,34 @@ ActionController::RoutingError (No route matches [GET] "/"):
    (91.0ms)  CREATE UNIQUE INDEX "unique_schema_migrations" ON "schema_migrations" ("version")
    (0.1ms)  SELECT version FROM "schema_migrations"
    (85.5ms)  INSERT INTO "schema_migrations" (version) VALUES ('0')
+   (0.1ms)  select sqlite_version(*)
+   (113.4ms)  CREATE TABLE "schema_migrations" ("version" varchar(255) NOT NULL) 
+   (0.0ms)  PRAGMA index_list("schema_migrations")
+   (101.1ms)  CREATE UNIQUE INDEX "unique_schema_migrations" ON "schema_migrations" ("version")
+   (0.1ms)  SELECT "schema_migrations"."version" FROM "schema_migrations" 
+
+
+Started GET "/" for 127.0.0.1 at 2012-04-16 11:15:17 +0200
+
+ActionController::RoutingError (No route matches [GET] "/"):
+  actionpack (3.2.2) lib/action_dispatch/middleware/debug_exceptions.rb:21:in `call'
+  actionpack (3.2.2) lib/action_dispatch/middleware/show_exceptions.rb:56:in `call'
+  railties (3.2.2) lib/rails/rack/logger.rb:26:in `call_app'
+  railties (3.2.2) lib/rails/rack/logger.rb:16:in `call'
+  actionpack (3.2.2) lib/action_dispatch/middleware/request_id.rb:22:in `call'
+  rack (1.4.1) lib/rack/methodoverride.rb:21:in `call'
+  rack (1.4.1) lib/rack/runtime.rb:17:in `call'
+  activesupport (3.2.2) lib/active_support/cache/strategy/local_cache.rb:72:in `call'
+  rack (1.4.1) lib/rack/lock.rb:15:in `call'
+  actionpack (3.2.2) lib/action_dispatch/middleware/static.rb:61:in `call'
+  railties (3.2.2) lib/rails/engine.rb:479:in `call'
+  railties (3.2.2) lib/rails/application.rb:220:in `call'
+  rack (1.4.1) lib/rack/content_length.rb:14:in `call'
+  railties (3.2.2) lib/rails/rack/log_tailer.rb:14:in `call'
+  rack (1.4.1) lib/rack/handler/webrick.rb:59:in `service'
+  /home/obale/.rvm/rubies/ruby-1.9.3-p125/lib/ruby/1.9.1/webrick/httpserver.rb:138:in `service'
+  /home/obale/.rvm/rubies/ruby-1.9.3-p125/lib/ruby/1.9.1/webrick/httpserver.rb:94:in `run'
+  /home/obale/.rvm/rubies/ruby-1.9.3-p125/lib/ruby/1.9.1/webrick/server.rb:191:in `block in start_thread'
+
+
+  Rendered /home/obale/.rvm/gems/ruby-1.9.3-p125@sigimera/gems/actionpack-3.2.2/lib/action_dispatch/middleware/templates/rescues/routing_error.erb within rescues/layout (3.3ms)

data/test/dummy/log/test.log

@@ -8483,3 +8483,31 @@ Interrupt: : rollback transaction
    (0.0ms)  rollback transaction
    (0.3ms)  begin transaction
    (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.1ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.1ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.1ms)  rollback transaction
+   (0.1ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction
+   (0.3ms)  begin transaction
+   (0.0ms)  rollback transaction

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: semantic-crawler
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-15 00:00:00.000000000 Z
+date: 2012-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -91,6 +91,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: grit
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -190,7 +206,6 @@ files:
 - MIT-LICENSE
 - README.rdoc
 - Rakefile
-- changelog.sh
 - exploitation/freebase.rb
 - lib/semantic_crawler.rb
 - lib/semantic_crawler/dbpedia.rb
@@ -263,7 +278,8 @@ files:
 - test/dummy/log/test.log
 - test/dummy/log/development.log
 homepage: https://github.com/obale/semantic_crawler
-licenses: []
+licenses:
+- MIT
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -276,7 +292,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4442978705485601093
+      hash: -4201444334486941347
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -285,10 +301,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4442978705485601093
+      hash: -4201444334486941347
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.21
+rubygems_version: 1.8.22
 signing_key: 
 specification_version: 3
 summary: SemanticCrawler is a ruby library that encapsulates data gathering from different

data/changelog.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-currentMerge=$(git log --merges master -1 --format=%h)
-previousMerge=$(git log --merges master -2 --format=%h | tail -n1)
-git log ${previousMerge}..${currentMerge}