pho 0.4.1 → 0.5

data/examples/sparql_construct_hash.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+require 'pho'
+
+query = <<-EOL
+PREFIX space: <http://purl.org/net/schemas/space/>
+PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+PREFIX foaf: <http://xmlns.com/foaf/0.1/>
+CONSTRUCT {
+  ?spacecraft foaf:name ?name;
+              space:agency ?agency;
+              space:mass ?mass.
+}
+WHERE {
+  ?launch space:launched "1969-07-16"^^xsd:date.
+  ?spacecraft space:launch ?launch;
+              foaf:name ?name;
+              space:agency ?agency;
+              space:mass ?mass.
+}
+EOL
+
+sparql_client = Pho::Sparql::SparqlClient.new("http://api.talis.com/stores/space/services/sparql")
+
+data = Pho::Sparql::SparqlHelper.constructToResourceHash(query, sparql_client)
+
+puts data.inspect()

data/examples/sparql_select.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'pho'
+
+query = <<-EOL
+PREFIX space: <http://purl.org/net/schemas/space/>
+SELECT ?s WHERE {
+   ?s a space:Spacecraft.
+}
+LIMIT 10
+EOL
+
+sparql_client = Pho::Sparql::SparqlClient.new("http://api.talis.com/stores/space/services/sparql")
+
+uris = Pho::Sparql::SparqlHelper.selectValues(query, sparql_client)
+
+uris.each do |uri|
+  puts uri
+end

data/lib/pho.rb

@@ -12,10 +12,15 @@ require 'pho/query_profile'
 require 'pho/status'
 require 'pho/facet'
 require 'pho/job'
+require 'pho/file_management'
 require 'pho/rdf_collection'
-require 'pho/rdf_json'
+require 'pho/file_manager'
+require 'pho/resource_hash'
+require 'pho/converter'
 require 'pho/changeset_builder'
 require 'pho/changeset'
+require 'pho/sparql'
+require 'pho/enrichment'
 
 if RUBY_VERSION < "1.8.7"
   class String

data/lib/pho/changeset.rb

@@ -1,9 +1,8 @@
 module Pho
 
-# Previous changeset
-# Identifier for changeset?
-# Blank nodes
-# Convenience functions for adding/removing lists?
+# TODO: assigning identifiers via changesets (Blank nodes on subject of change and subject)  
+# TODO: assigning identifiers to a changeset docs
+# TODO: linking together changesets using either nodeID or an identifier
       
   require 'uri'
   
@@ -43,15 +42,19 @@ module Pho
         #Create an RDF/XML fragment describing this Statement        
         def to_rdf()
           rdf = "<rdf:Statement>"
-          rdf << "  <rdf:subject rdf:resource=\"#{@subject}\"/>"
+          rdf << write_subject()
           rdf << "  <rdf:predicate rdf:resource=\"#{@predicate}\"/>"
           rdf << write_object()
           rdf << "</rdf:Statement>"
           return rdf          
         end
-       
+               
         private
-        
+       
+         def write_subject()
+           return "  <rdf:subject rdf:resource=\"#{@subject}\"/>"
+         end
+         
          #  subject:: URI of subject of triple
          #  predicate:: URI of predicate of triple
          #  object:: object value of triple (may be URI or literal)
@@ -102,8 +105,16 @@ module Pho
        
        protected
         def write_object()
-          #FIXME language and datatype
-          "<rdf:object>#{@object}</rdf:object>"
+          tag = ""
+          if @datatype != nil
+            tag = "<rdf:object rdf:datatype=\"#{@datatype}\">"
+          elsif @language != nil
+            tag = "<rdf:object xml:lang=\"#{@language}\">"
+          else
+            tag = "<rdf:object>" 
+          end
+          tag << "#{@object}</rdf:object>"          
+          return tag
         end
     end
 
@@ -127,6 +138,10 @@ module Pho
        end
       
        protected
+        def write_subject()
+           return "  <rdf:subject rdf:resource=\"#{@subject}\"/>"
+        end
+       
         def write_object()
           "<rdf:object rdf:resource=\"#{@object}\"/>"
         end

data/lib/pho/changeset_builder.rb

@@ -15,14 +15,14 @@ module Pho
       #
       #Returns an array of Changeset objects
       #
-      # before:: triple hash describing current state of the resource
-      # after:: triple hash describing updated state of the resource
+      # before:: resource hash describing current state of the resource
+      # after:: resource hash describing updated state of the resource
       # creator_name:: name of the creator of the changes
       # change_reason:: description of why the changes are being made      
       def ChangesetBuilder.build_batch(before, after, creator_name=nil, change_reason=nil)
 
-        removals = Pho::RDF_JSON::SetAlgebra.minus(before, after)
-        additions = Pho::RDF_JSON::SetAlgebra.minus(after, before)
+        removals = Pho::ResourceHash::SetAlgebra.minus(before, after)
+        additions = Pho::ResourceHash::SetAlgebra.minus(after, before)
         
         batch = Array.new
         
@@ -58,13 +58,13 @@ module Pho
       #The method will return a single Changeset object.
       #
       # subject_of_change:: uri of the resource being updated
-      # before:: triple hash describing current state of the resource
-      # after:: triple hash describing updated state of the resource
+      # before:: resource hash describing current state of the resource
+      # after:: resource hash describing updated state of the resource
       # creator_name:: name of the creator of the changes
       # change_reason:: description of why the changes are being made
       def ChangesetBuilder.build(subject_of_change, before, after, creator_name=nil, change_reason=nil)
-        removals = Pho::RDF_JSON::SetAlgebra.minus(before, after)
-        additions = Pho::RDF_JSON::SetAlgebra.minus(after, before)
+        removals = Pho::ResourceHash::SetAlgebra.minus(before, after)
+        additions = Pho::ResourceHash::SetAlgebra.minus(after, before)
         
         cs = Pho::Update::Changeset.new(subject_of_change, creator_name, change_reason) do |cs|
             cs.add_removals( create_statements(removals) )
@@ -75,9 +75,9 @@ module Pho
         
       end  
       
-      #Takes an triple hash and serializes it as an array of Pho::Update::Statement objects
+      #Takes a resource hash and serializes it as an array of Pho::Update::Statement objects
       #
-      # triples:: a hash of triples, conforming to RDF-in-JSON structure
+      # triples:: a resource hash, conforming to RDF-in-JSON structure
       def ChangesetBuilder.create_statements(triples)
         statements = Array.new
         triples.each do |uri, properties|

data/lib/pho/converter.rb

@@ -0,0 +1,74 @@
+module Pho
+  
+  module ResourceHash
+
+    #TODO reading from streams
+    #TODO convert into other formats
+    #TODO wrap Redland exceptions. Parser/Serializer contruction as well as parsing errors
+    
+    require 'rdf/redland'
+    
+    #Class for converting to and from resource hashes 
+    class Converter
+      
+      #Parse JSON structured according to the RDF-in-JSON specification into 
+      #a Ruby resource hash. Simply invokes the JSON parser.
+      #
+      # json:: valid RDF-in-JSON
+      def Converter.parse_json(json)
+        return JSON.parse(json)
+      end
+      
+      #Parse a string containing RDF/XML into a resource hash
+      #
+      # rdfxml: a String containing RDF/XML
+      def Converter.parse_rdfxml(rdfxml, base_uri)
+        return Converter.parse(rdfxml, base_uri, "rdfxml")
+      end
+
+      #Parse a string containing N-Triples into a resource hash
+      #
+      # ntriples:: a String containing N-Triples
+      def Converter.parse_ntriples(ntriples, base_uri)
+        return Converter.parse(ntriples, base_uri, "ntriples")
+      end
+
+      #Parse a string containing Turtle into a resource hash
+      #
+      # ntriples:: a String containing Turtle
+      def Converter.parse_turtle(turtle, base_uri)
+        return Converter.parse(turtle, base_uri, "turtle")
+      end
+                              
+      #Convert specified format into a ResourceHash
+      #
+      # format:: one of rdfxml, ntriples, turtle
+      # data:: String containing the data to be parsed
+      # base_uri:: base uri of the data
+      def Converter.parse(data, base_uri, format="rdfxml")
+        model = Redland::Model.new()
+        case format
+          when "rdfxml" then mime="application/rdf+xml"
+          when "json" then mime="application/json"
+          else mime=""     
+        end
+        
+        parser = Redland::Parser.new(format, mime)
+        parser.parse_string_into_model(model, data, base_uri)
+        serializer = Redland::Serializer.new( "json", "application/json" )
+        json = serializer.model_to_string(Redland::Uri.new(base_uri), model)
+        return Converter.parse_json( json )        
+      end
+
+      #Serialize a resource hash as RDF-in-JSON
+      #
+      # hash:: the resource hash to serialize      
+      def Converter.serialize_json(hash)
+          return JSON.dump(hash)  
+      end
+            
+      
+    end
+            
+  end  
+end

data/lib/pho/enrichment.rb

@@ -0,0 +1,81 @@
+module Pho
+    
+  #Enrichment module. Provides classes and mixins for enriching data held in Platform stores using
+  #external SPARQL endpoints and services
+  module Enrichment
+    
+    class ResourceEnricher
+      
+      attr_reader :store
+      attr_reader :sparql_client
+      
+      # Constructor
+      #  store:: the store containing resource(s) to enrich
+      #  sparql_client:: SparqlClient object to be used as client for retrieving enrichments
+      def initialize(store, sparql_client)
+          @store = store
+          @sparql_client = sparql_client  
+      end
+
+      #Enrich a store against itself
+      #
+      #For the common case where a store needs to be enriched against itself by inferring new 
+      #data from old using a CONSTRUCT query
+      #
+      # store:: the store to be updated
+      # query:: the SPARQL CONSTRUCT query to be used to infer the new data
+      def ResourceEnricher.infer(store, query)
+        enricher = ResourceEnricher.new( store, store.sparql_client() )
+        return enricher.merge( query )
+      end 
+            
+      #Execute the provided query against the configured SPARQL endpoint and store the results in 
+      #the Platform Store
+      #
+      # query:: the SPARQL CONSTRUCT or DESCRIBE query to execute
+      def merge(query)
+        resp = @sparql_client.query( query, "application/rdf+xml" )
+        if resp.status != 200
+           raise "Unable to execute query. Response: #{resp.status} #{resp.reason} #{resp.content}"  
+        end
+        resp = @store.store_data( resp.content )
+        return resp        
+      end
+      
+      #TODO: optimize POSTs back to the Platform, to deal with large number of resources, e.g. batching
+      
+      # Enrich a store with data extracted from a SPARQL endpoint.
+      #
+      # The locator_query is used to find resources in the Platform Store that should be enriched. The query
+      # should be a SPARQL SELECT query that returns the data to be used to parameterize the enrichment_query.
+      #
+      # For each query result, the enrichment_query will be submitted to the configured SPARQL endpoint, after 
+      # first interpolating the string, providing the query result bindings as parameters. (See SparqlHelper.apply_initial_bindings 
+      # and SparqlHelper.results_to_query_bindings 
+      #
+      # If successful, the result of each enrichment query will then be pushed back into the Platform Store by 
+      # posting the results of the query to the metabox. Enrichment queries should therefore be CONSTRUCT or 
+      # DESCRIBE queries. The SPARQL protocol request will be sent with an Accept header of application/rdf+xml
+      #
+      #  locator_query:: query to locate resources to be enriched
+      #  enrichment_query:: query to be used to enrich the resource
+      def enrich(locator_query, enrichment_query)
+          results = Pho::Sparql::SparqlHelper.select(locator_query, @store.sparql_client() )
+          bindings = Pho::Sparql::SparqlHelper.results_to_query_bindings(results)
+          bindings.each do |binding|
+            bound_query = Pho::Sparql::SparqlHelper.apply_initial_bindings(enrichment_query, binding)
+
+            result = @sparql_client.query(bound_query, "application/rdf+xml")
+            
+            #TODO error handling and callbacks            
+            if result.status == 200
+              resp = @store.store_data( result.content )  
+            end        
+            
+          end
+      end
+      
+    end
+    
+  end
+end

data/lib/pho/etags.rb

@@ -1,5 +1,6 @@
 require 'yaml'
 
+#TODO put together proper test suite for this
 module Pho
   
   #Simple mechanism for managing etags

data/lib/pho/field_predicate_map.rb

@@ -116,7 +116,12 @@ module Pho
         
         json = JSON.parse( resp.content )
         
-        label = json[fpmap_uri]["http:\/\/www.w3.org\/2000\/01\/rdf-schema#label"][0]["value"]
+        labels = json[fpmap_uri]["http:\/\/www.w3.org\/2000\/01\/rdf-schema#label"]
+        label = ""
+        if labels != nil
+          label = labels[0]["value"]
+        end
+        
         fpmap = FieldPredicateMap.new(fpmap_uri, label)
         
         mapped_properties = json[fpmap_uri]["http:\/\/schemas.talis.com\/2006\/frame\/schema#mappedDatatypeProperty"]

data/lib/pho/file_management.rb

@@ -0,0 +1,102 @@
+module Pho
+  
+  module FileManagement
+
+    #TODO: move files into hidden directory
+    #TODO: add directory traversal
+    class AbstractFileManager
+    
+      attr_reader :dir
+      attr_reader :store
+        
+      OK = "ok".freeze
+      FAIL = "fail".freeze
+      
+      def initialize(store, dir, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
+        @store = store
+        @dir = dir
+        @sleep = sleep
+        @ok_suffix = ok_suffix
+        @fail_suffix = fail_suffix
+      end
+  
+      #Store all files that match the file name in directory
+      def store()
+        files_to_store = new_files()
+        files_to_store.each do |filename|
+          file = File.new(filename)
+          store_file(file, filename)
+        end    
+      end
+  
+      #Retry anything known to have failed
+      def retry_failures()
+        retries = failures()
+        retries.each do |filename|
+          File.delete( get_fail_file_for(filename) )
+          #store it
+          file = File.new(filename)
+          store_file(file, filename)      
+        end
+      end
+            
+      #Reset the directory to clear out any previous statuses
+      #Store can  also be reset at the same time: use with care!
+      def reset(reset_store=false)
+        Dir.glob( File.join(@dir, "*.#{@fail_suffix}") ).each do |file|
+          File.delete(file)
+        end
+        Dir.glob( File.join(@dir, "*.#{@ok_suffix}") ).each do |file|
+          File.delete(file)
+        end         
+      end
+                    
+      #List failures
+      def failures()
+        fails = Array.new
+        Dir.glob( File.join(@dir, "*") ).each do |file|
+          if File.extname(file) != ".#{@fail_suffix}" && File.extname(file) != ".#{@ok_suffix}"
+            fails << file if File.exists?( get_fail_file_for(file) )  
+          end          
+        end
+        return fails
+      end
+      
+      #List successes
+      def successes()
+        successes = Array.new
+        Dir.glob( File.join(@dir, "*") ).each do |file|
+          if File.extname(file) != ".#{@fail_suffix}" && File.extname(file) != ".#{@ok_suffix}"
+             successes << file if File.exists?( get_ok_file_for(file) )
+          end
+        end
+        return successes
+      end
+            
+      #Summarize the state of the collection to the provied IO object
+      #Creates a simple report
+      def summary()
+         failures = failures()
+         successes = successes()
+         newfiles = new_files()
+         total = failures.size + successes.size + newfiles.size
+         summary = "#{@dir} contains #{total} files: #{successes.size} stored, #{failures.size} failed, #{newfiles.size} new"
+         return summary     
+      end
+      
+      def get_fail_file_for(filename)
+        ext = File.extname(filename)
+        return filename.gsub(/#{ext}/, ".#{@fail_suffix}")              
+      end
+      
+      def get_ok_file_for(filename)
+        ext = File.extname(filename)        
+        return filename.gsub(/#{ext}/, ".#{@ok_suffix}")
+      end
+                 
+    end
+    
+  end
+  #end file Module  
+  
+end