semantic-crawler 0.2.0 → 0.3.0

data/.gitignore

@@ -0,0 +1,14 @@
+/doc/
+/rdoc/
+*.swp
+/.bundle/
+/.yardoc
+/log/*.log
+/log/*.log.*
+/pkg/
+/test.sh
+test/dummy/log/*.log
+test/dummy/tmp/
+test/dummy/.sass-cache
+/coverage
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--profile
+#--format documentation

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+    - 1.8.7
+    - 1.9.3

data/.yardopts

@@ -0,0 +1,5 @@
+--title "Semantic Crawler Library for Ruby"
+--output-dir doc/yard
+--markup rdoc
+--readme README.rdoc
+-

data/Gemfile

@@ -0,0 +1,18 @@
+source "http://rubygems.org"
+
+# Declare your gem's dependencies in semantic_crawler.gemspec.
+# Bundler will treat runtime dependencies like base dependencies, and
+# development dependencies will be added by default to the :development group.
+gemspec
+
+# jquery-rails is used by the dummy application
+gem "jquery-rails"
+
+# 
+# Declare any dependencies that are still in development here instead of in
+# your gemspec. These might include edge Rails or gems from your path or
+# Git. Remember to move these dependencies to your gemspec before releasing
+# your gem to rubygems.org.
+
+# To use debugger
+# gem 'ruby-debug'

data/README.rdoc

@@ -1,4 +1,4 @@
-= SemanticCrawler {<img src="https://secure.travis-ci.org/obale/semantic_crawler.png?branch=master"/>}[http://travis-ci.org/#!/obale/semantic_crawler]
+= SemanticCrawler 
 
 This project encapsulates data gathering from different sources.
 It simplifies the extension of internal data with public available
@@ -8,15 +8,15 @@ to bypass complex NLP (natural language processing).
 
 == Supported Sources
 
-* CIA Factbook RDF Dump, see http://www4.wiwiss.fu-berlin.de/factbook/directory/countries
-* GDACS (see http://gdacs.org)
-* FAO - Food and Agriculture Organization of the United Nations (see http://www.fao.org)
-* LinkedGeoData - LGD (see http://linkedgeodata.org)
-* [Started] Freebase (see http://freebase.com)
+* {Geonames}[http://www.geonames.org/]
+* {CIA Factbook RDF Dump}[http://www4.wiwiss.fu-berlin.de/factbook/directory/countries]
+* {FAO - Food and Agriculture Organization of the United Nations}[http://www.fao.org]
+* {LinkedGeoData - LGD}[http://linkedgeodata.org]
+* {GDACS}[http://gdacs.org]
+* [Started] {Freebase}[http://freebase.com]
 
 === TODO
 
-* Geonames
 * DBPedia
 * Different Government Sources
 
@@ -45,8 +45,25 @@ Or from source:
 
 == Examples
 
+This examples are only a short outline how to use the library. For more
+information read the documentation or look into the source code. To use the
+library include or execute the following line:
+
     >> require "semantic_crawler"
 
+=== GeoNames
+
+The GeoNames module is able to return a Factbook::Country and Fao::Country
+module on the base of input GPS coordinates (lat/long).
+
+    >> @innsbruck = SemanticCrawler::GeoNames::Country.new(47.271338, 11.395333)
+    >> articles = @innsbruck.get_wikipedia_articles
+    >> articles.each do |article|
+    >>      puts article.wikipedia_url
+    >> end
+    >> factbook_obj = @innsbruck.get_factbook_country
+    >> fao_obj = @innsbruck.get_fao_country
+
 === Factbook
 
 Fetch Factbook information about Austria:
@@ -90,7 +107,7 @@ Country information from {FAO}[http://www.fao.org]:
 
 === LinkedGeoData
 
-Geo information from {LinkedGeoData}[http://linkedgeodata.org]
+Geo information from {LinkedGeoData}[http://linkedgeodata.org]:
 
     >> # All nodes around the center of dresden, in a radius of 1000m
     >> @dresden = SemanticCrawler::LinkedGeoData::RelevantNodes.new(51.033333, 13.733333, 1000)

data/Rakefile

@@ -21,7 +21,6 @@ RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.options << '--line-numbers'
   rdoc.options << '--main=README.rdoc'
   rdoc.rdoc_files.include('README.rdoc')
-  rdoc.rdoc_files.include('CHANGELOG.rdoc')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 

data/changelog.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+currentMerge=$(git log --merges master -1 --format=%h)
+previousMerge=$(git log --merges master -2 --format=%h | tail -n1)
+git log ${previousMerge}..${currentMerge}

data/exploitation/freebase.rb

@@ -0,0 +1,13 @@
+require 'cgi'
+require 'httparty'
+require 'json'
+require 'pp'
+
+#query = [{'id' => '/en/austria', 'name' => nil, 'type' => '/location/country', '*' => [{}] }]
+query = [{'id' => '/en/austria', 'type' => '/type/property' }]
+query_envelope = {'query' => query }
+service_url = 'http://api.freebase.com/api/service/mqlread'
+url = service_url + '?query=' + CGI::escape(query_envelope.to_json)
+
+response = HTTParty.get(url, :format => :json)
+puts response

data/lib/semantic_crawler.rb

@@ -1,6 +1,7 @@
 # encoding: UTF-8
 
 require "httparty"
+require "geonames"
 
 # The top level module contains the different data sources
 # as sub-modules. Currently there are the following modules
@@ -38,6 +39,7 @@ require "semantic_crawler/dbpedia"
 
 # GeoNames.org - module: GeoNames
 require "semantic_crawler/geo_names"
+require "semantic_crawler/geo_names/country"
 
 # LinkedGeoData.org - module: LinkedGeoData
 require "semantic_crawler/linked_geo_data"

data/lib/semantic_crawler/factbook/country.rb

@@ -118,6 +118,7 @@ module SemanticCrawler
                 end
             end
 
+            # @return [String] The document serialized as XML
             def xml_document
                 @doc.to_s
             end

data/lib/semantic_crawler/fao/country.rb

@@ -3,6 +3,7 @@ module SemanticCrawler
         # Represents Food and Agriculture information about one country.
         class Country
 
+            # The URI prefix of the fao country object
             @@URI_PREFIX = "http://www.fao.org/countryprofiles/geoinfo/geopolitical/data/"
 
             # Namespace hash
@@ -35,11 +36,13 @@ module SemanticCrawler
             end
 
             # The dbpedia identifier (from fao:codeDBPediaID)
+            # @return [String]
             def code_dbpedia_id
                 query_root_node("fao:codeDBPediaID/text()", @@NAMESPACES).to_s
             end
 
             # Links to additional information (from owl:sameAs)
+            # @return [Array<String>]
             def same_as
                 returnLinks = []
                 links = query_root_node("owl:sameAs/@rdf:resource", @@NAMESPACES)
@@ -50,63 +53,75 @@ module SemanticCrawler
             end
 
             # The type as URL of this entity (from rdf:type)
+            # @return [String]
             def type_url
                 query_root_node("rdf:type/@rdf:resource", @@NAMESPACES).to_s
             end
 
             # The maximum latitude (from fao:hasMaxLatitude)
+            # @return [String]
             def max_latitude
                 query_root_node("fao:hasMaxLatitude/text()", @@NAMESPACES).to_s
             end
 
             # The maximum longitude (from fao:hasMaxLongitude)
+            # @return [String]
             def max_longitude
                 query_root_node("fao:hasMaxLongitude/text()", @@NAMESPACES).to_s
             end
 
             # The minimum latitude (from fao:hasMinLatitude)
+            # @return [String]
             def min_latitude
                 query_root_node("fao:hasMinLatitude/text()", @@NAMESPACES).to_s
             end
 
             # The minimum longitude (from fao:hasMinLongitude)
+            # @return [String]
             def min_longitude
                 query_root_node("fao:hasMinLongitude/text()", @@NAMESPACES).to_s
             end
 
             # Human readable description about the land area (from fao:landAreaNotes)
+            # @return [String]
             def land_area_notes
                 query_root_node("fao:landAreaNotes/text()", @@NAMESPACES).to_s
             end
 
             # Land area total value (from fao:landAreaTotal)
+            # @return [String]
             def land_area_total
                 query_root_node("fao:landAreaTotal/text()", @@NAMESPACES).to_s
             end
 
             # Land area unit (from fao:landAreaUnit)
+            # @return [String]
             def land_area_unit
                 query_root_node("fao:landAreaUnit/text()", @@NAMESPACES).to_s
             end
 
             # Land area year (from fao:landAreaYear)
+            # @return [String]
             def land_area_year
                 query_root_node("fao:landAreaYear/text()", @@NAMESPACES).to_s
             end
 
             # The currency name.
-            # @param [Lang] The language in which the currency name should be returned
-            def name_currency(lang)
+            # @param [String] The language in which the currency name should be returned
+            # @return [String]
+            def name_currency(lang = 'en')
                 query_root_node("fao:nameCurrency[@xml:lang='#{lang}']/text()", @@NAMESPACES).to_s
             end
 
             # The official country name
-            # @param [Lang] The language in which the official name should be returned
-            def official_name(lang)
+            # @param [String] The language in which the official name should be returned
+            # @return [String]
+            def official_name(lang = 'en')
                 query_root_node("fao:nameOfficial[@xml:lang='#{lang}']/text()", @@NAMESPACES).to_s
             end
 
             # Classification of this country as name (from fao:isInGroup)
+            # @return [Array<String>]
             def is_in_group_name
                 returnGroup = []
                 group = query_root_node("fao:isInGroup/@rdf:resource", @@NAMESPACES)
@@ -120,6 +135,7 @@ module SemanticCrawler
 
             # Classification of this country as dereferenceable URL (from
             # fao:isInGroup)
+            # @return [Array<String>]
             def is_in_group_url
                 returnGroup = []
                 group = query_root_node("fao:isInGroup/@rdf:resource", @@NAMESPACES)
@@ -133,6 +149,7 @@ module SemanticCrawler
 
             # Returns all countries that share a boarder with this country (as
             # dereferencable URL - from fao:hasBorderWith)
+            # @return [Array<String>]
             def has_boarder_with_url
                 returnGroup = []
                 group = query_root_node("fao:hasBorderWith/@rdf:resource", @@NAMESPACES)
@@ -146,6 +163,7 @@ module SemanticCrawler
 
             # Returns all countries that share a boarder with this country (as
             # name)
+            # @return [Array<String>]
             def has_boarder_with_name
                 returnGroup = []
                 group = query_root_node("fao:hasBorderWith/@rdf:resource", @@NAMESPACES)
@@ -158,36 +176,43 @@ module SemanticCrawler
             end
 
             # Population notes (from fao:populationNotes)
+            # @return [String]
             def population_notes
                 query_root_node("fao:populationNotes/text()", @@NAMESPACES).to_s
             end
 
             # Population total (from fao:populationTotal)
+            # @return [String]
             def population_total
                 query_root_node("fao:populationTotal/text()", @@NAMESPACES).to_s
             end
 
             # Population unit (from fao:populationUnit)
+            # @return [String]
             def population_unit
                 query_root_node("fao:populationUnit/text()", @@NAMESPACES).to_s
             end
 
             # Population year (from fao:populationYear)
+            # @return [String]
             def population_year
                 query_root_node("fao:populationYear/text()", @@NAMESPACES).to_s
             end
 
             # Entity is valid since (from fao:validSince)
+            # @return [String]
             def valid_since
                 query_root_node("fao:validSince/text()", @@NAMESPACES).to_s
             end
 
             # Entity is valid until (from fao:validUntil)
+            # @return [String]
             def valid_until
                 query_root_node("fao:validUntil/text()", @@NAMESPACES).to_s
             end
 
             # Executes a xpath query with optional a hash with namespaces
+            # @return [String]
             def query_root_node(xpath_query, namespaces = {})
                 if !@root_node.nil?
                     @root_node.xpath(xpath_query, namespaces)
@@ -195,6 +220,7 @@ module SemanticCrawler
             end
 
             # Outputs the document as XML
+            # @return [String] The document serialized as XML
             def xml_document
                 @root_node.to_s
             end

data/lib/semantic_crawler/gdacs/feed.rb

@@ -11,6 +11,7 @@ module SemanticCrawler
         # * Volcanoes
         class Feed
 
+            # XML namespaces used for the parsing process
             @@NAMESPACES = {
                 "atom" => "http://www.w3.org/2005/Atom"
             }

data/lib/semantic_crawler/geo_names/country.rb

@@ -0,0 +1,48 @@
+module SemanticCrawler
+    module GeoNames
+        # Represents Food and Agriculture information about one country.
+        class Country
+
+            # @attribute [r]
+            # The input latitude
+            attr_reader :latitude
+
+            # @attribute [r]
+            # The input longitude
+            attr_reader :longitude
+
+            def initialize(new_latitude, new_longitude)
+                @latitude = new_latitude
+                @longitude = new_longitude
+            end
+
+            # Returns an ISO 3166-1 alpha-2 country code
+            def get_country_code
+                Geonames::WebService.country_code @latitude, @longitude
+            end
+
+            # Returns the name of the country
+            def get_country_name
+                subdivision = Geonames::WebService.country_subdivision @latitude, @longitude
+                subdivision.country_name
+            end
+
+            # Returns a Wikipedia object that contains a link to the article,
+            # summary and many more fields.
+            def get_wikipedia_articles
+                Geonames::WebService.find_nearby_wikipedia :lat => @latitude, :long => @longitude
+            end
+
+            # @return [SemanticCrawler::Factbook::Country] A Factbook country object
+            def get_factbook_country
+                SemanticCrawler::Factbook::Country.new get_country_name
+            end
+
+            # @return [SemanticCrawler::Fao::Country] A Food and Agriculture country object
+            def get_fao_country
+                SemanticCrawler::Fao::Country.new get_country_name
+            end
+
+        end
+    end
+end

data/lib/semantic_crawler/linked_geo_data/relevant_node.rb

@@ -23,33 +23,39 @@ module SemanticCrawler
             end
 
             # geo:lat
+            # @return [String]
             def latitude
-                query_root_node("geo:lat/text()", @@NAMESPACES)
+                query_root_node("geo:lat/text()", @@NAMESPACES).to_s
             end
 
             # geo:long
+            # @return [String]
             def longitude
-                query_root_node("geo:long/text()", @@NAMESPACES)
+                query_root_node("geo:long/text()", @@NAMESPACES).to_s
             end
 
             # The type as URL
+            # @return [String]
             def type
-                query_root_node("rdf:type/@rdf:resource", @@NAMESPACES)
+                query_root_node("rdf:type/@rdf:resource", @@NAMESPACES).to_s
             end
 
             # Comment about the entity
+            # @return [String]
             def note
-                query_root_node("lgdp:note/text()", @@NAMESPACES)
+                query_root_node("lgdp:note/text()", @@NAMESPACES).to_s
             end
 
             # Data Source
+            # @return [String]
             def created_by
-                query_root_node("lgdp:created_by/text()", @@NAMESPACES)
+                query_root_node("lgdp:created_by/text()", @@NAMESPACES).to_s
             end
 
             # User link
+            # @return [String]
             def contributor
-                query_root_node("lgdo:contributor/@rdf:resource", @@NAMESPACES)
+                query_root_node("lgdo:contributor/@rdf:resource", @@NAMESPACES).to_s
             end
 
             # Link to way 
@@ -58,33 +64,39 @@ module SemanticCrawler
             end
 
             # Street name
+            # @return [String]
             def has_street
-                query_root_node("lgdo:hasStreet/text()", @@NAMESPACES)
+                query_root_node("lgdo:hasStreet/text()", @@NAMESPACES).to_s
             end
 
             # Postal code
+            # @return [String]
             def has_postal_code
-                query_root_node("lgdo:hasPostalCode/text()", @@NAMESPACES)
+                query_root_node("lgdo:hasPostalCode/text()", @@NAMESPACES).to_s
             end
 
             # House number
+            # @return [String]
             def has_house_number
-                query_root_node("lgdo:hasHouseNumber/text()", @@NAMESPACES)
+                query_root_node("lgdo:hasHouseNumber/text()", @@NAMESPACES).to_s
             end
 
             # Country in ISO 3166-1 alpha-2
+            # @return [String]
             def has_country
-                query_root_node("lgdo:hasCountry/text()", @@NAMESPACES)
+                query_root_node("lgdo:hasCountry/text()", @@NAMESPACES).to_s
             end
 
             # City name
+            # @return [String]
             def has_city
-                query_root_node("lgdo:hasCity/text()", @@NAMESPACES)
+                query_root_node("lgdo:hasCity/text()", @@NAMESPACES).to_s
             end
 
             # Is this place wheelchair friendly?
+            # @return [String]
             def wheelchair
-                query_root_node("lgdo:wheelchair/@rdf:resource", @@NAMESPACES)
+                query_root_node("lgdo:wheelchair/@rdf:resource", @@NAMESPACES).to_s
             end
 
             # Query the root_node
@@ -95,6 +107,7 @@ module SemanticCrawler
             end
 
             # Outputs the document as XML
+            # @return [String] The document serialized as XML
             def xml_document
                 @root_node.to_s
             end