pho 0.4.1 → 0.5

data/lib/pho/file_manager.rb

@@ -0,0 +1,61 @@
+module Pho
+
+  require 'mime/types'
+    
+  module FileManagement
+
+    class FileManager < AbstractFileManager
+   
+      def initialize(store, dir, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
+        super(store, dir, ok_suffix, fail_suffix, sleep)
+      end
+                
+      #List files being managed, i.e. everything not .ok or .fail
+      def list()          
+          files = []
+          Dir.glob( File.join(@dir, "*") ) do |file|
+            if File.extname(file) != ".#{@ok_suffix}" && File.extname(file) != ".#{@fail_suffix}" 
+              files << file
+            end
+          end  
+          return files
+      end
+                      
+      #List any new files in the directory
+      def new_files()
+        newfiles = Array.new
+        Dir.glob( File.join(@dir, "*") ) do |file|
+          
+          if File.extname(file) != ".#{@ok_suffix}" && File.extname(file) != ".#{@fail_suffix}" 
+            ok_file = get_ok_file_for(file)
+            fail_file = get_fail_file_for(file)
+            if !( File.exists?(ok_file) or File.exists?(fail_file) )
+              newfiles << file          
+            end
+            
+          end
+          
+        end
+        return newfiles
+      end
+
+       
+      protected
+  
+      def store_file(file, filename)
+        response = @store.upload_item(file, MIME::Types.type_for(filename)[0].to_s )
+        if (response.status < 300 )
+          File.open(get_ok_file_for(filename), "w") do |file|
+            file.print( "OK" )            
+          end
+        else
+          File.open(get_fail_file_for(filename), "w") do |file|
+            YAML::dump(response, file)
+          end            
+        end      
+      end  
+         
+    end
+    
+  end
+end

data/lib/pho/rdf_collection.rb

@@ -1,129 +1,63 @@
 module Pho
   
-  attr_reader :dir
-  attr_reader :store
-  
-  # Provides a simple mechanism for managing a directory of RDF/XML documents
-  # and uploading them to platform store.
-  #
-  # Allows a collection to be mirrored into the platform
-  class RDFCollection
-    
-    RDF = "rdf".freeze
-    OK = "ok".freeze
-    FAIL = "fail".freeze
-    
-    def initialize(store, dir, rdf_suffix=RDF, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
-      @store = store
-      @dir = dir
-      @sleep = sleep
-      @rdf_suffix = rdf_suffix
-      @ok_suffix = ok_suffix
-      @fail_suffix = fail_suffix
-    end
+  module FileManagement
 
-    #Store all files that match the file name in directory
-    def store()
-      files_to_store = new_files()
-      files_to_store.each do |filename|
-        file = File.new(filename)
-        store_file(file, filename)
-      end    
-    end
-
-    #Retry anything known to have failed
-    def retry_failures()
-      retries = failures()
-      retries.each do |filename|
-        File.delete( get_fail_file_for(filename) )
-        #store it
-        file = File.new(filename)
-        store_file(file, filename)      
-      end
-    end
-          
-    #Reset the directory to clear out any previous statuses
-    #Store can  also be reset at the same time: use with care!
-    def reset(reset_store=false)
-      Dir.glob( File.join(@dir, "*.#{@fail_suffix}") ).each do |file|
-        File.delete(file)
-      end
-      Dir.glob( File.join(@dir, "*.#{@ok_suffix}") ).each do |file|
-        File.delete(file)
-      end         
-    end
-            
-    #List files being managed
-    def list()
-        return Dir.glob( File.join(@dir, "*.#{@rdf_suffix}") )  
-    end
-    
-    #List failures
-    def failures()
-      fails = Array.new
-      Dir.glob( File.join(@dir, "*.#{@fail_suffix}") ).each do |file|
-        fails << file.gsub(/\.#{@fail_suffix}/, ".#{@rdf_suffix}")
-      end
-      return fails
-    end
-    
-    #List successes
-    def successes()
-      successes = Array.new
-      Dir.glob( File.join(@dir, "*.#{@ok_suffix}") ).each do |file|
-        successes << file.gsub(/\.#{@ok_suffix}/, ".#{@rdf_suffix}")
-      end
-      return successes
-    end
-    
-    #List any new files in the directory
-    def new_files()
-      newfiles = Array.new
-      Dir.glob( File.join(@dir, "*.#{@rdf_suffix}") ) do |file|
-        ok_file = get_ok_file_for(file)
-        fail_file = get_fail_file_for(file)
-        if !( File.exists?(ok_file) or File.exists?(fail_file) )
-          newfiles << file          
+      # Provides a simple mechanism for managing a directory of RDF/XML documents
+      # and uploading them to platform store.
+      #
+      # Allows a collection to be mirrored into the platform
+      class RDFManager < AbstractFileManager
+      
+        RDF = "rdf".freeze
+        
+        def initialize(store, dir, rdf_suffix=RDF, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
+          super(store, dir, ok_suffix, fail_suffix, sleep)
+          @rdf_suffix = rdf_suffix
         end
-      end
-      return newfiles
-    end
     
-    #Summarize the state of the collection to the provied IO object
-    #Creates a simple report
-    def summary()
-       failures = failures()
-       successes = successes()
-       newfiles = new_files()
-       total = failures.size + successes.size + newfiles.size
-       summary = "#{@dir} contains #{total} files: #{successes.size} stored, #{failures.size} failed, #{newfiles.size} new"
-       return summary     
-    end
-    
-    def get_fail_file_for(filename)
-      return filename.gsub(/\.#{@rdf_suffix}/, ".#{@fail_suffix}")              
-    end
-    
-    def get_ok_file_for(filename)
-      return filename.gsub(/\.#{@rdf_suffix}/, ".#{@ok_suffix}")
-    end
-     
-    private
-
-    def store_file(file, filename)
-      response = @store.store_file(file)
-      if (response.status < 300 )
-        File.open(get_ok_file_for(filename), "w") do |file|
-          file.print( "OK" )            
+                
+        #List files being managed
+        def list()
+            return Dir.glob( File.join(@dir, "*.#{@rdf_suffix}") )  
         end
-      else
-        File.open(get_fail_file_for(filename), "w") do |file|
-          YAML::dump(response, file)
-        end            
-      end      
-    end  
-    
+                        
+        #List any new files in the directory
+        def new_files()
+          newfiles = Array.new
+          Dir.glob( File.join(@dir, "*.#{@rdf_suffix}") ) do |file|
+            ok_file = get_ok_file_for(file)
+            fail_file = get_fail_file_for(file)
+            if !( File.exists?(ok_file) or File.exists?(fail_file) )
+              newfiles << file          
+            end
+          end
+          return newfiles
+        end
+
          
+        protected
+    
+        def store_file(file, filename)
+          response = @store.store_file(file)
+          if (response.status < 300 )
+            File.open(get_ok_file_for(filename), "w") do |file|
+              file.print( "OK" )            
+            end
+          else
+            File.open(get_fail_file_for(filename), "w") do |file|
+              YAML::dump(response, file)
+            end            
+          end      
+        end  
+        
+             
+      end
+              
   end
-    
+  
+  #Deprecated. Use Pho::FileManangement::RDFManager instead
+  class RDFCollection < Pho::FileManagement::RDFManager
+    #for backwards compatibility
+  end
+  
 end  

data/lib/pho/{rdf_json.rb → resource_hash.rb}

@@ -1,11 +1,10 @@
 module Pho
 
-  #TODO
-  #blank nodes
+  #TODO blank nodes
   
-  #Module providing code for manipulating triple hashes structured according 
+  #Module providing code for manipulating resource hashes structured according 
   #to the RDF in JSON spec
-  module RDF_JSON
+  module ResourceHash
     
     #Class providing set algebra methods over triple hashes
     class SetAlgebra

data/lib/pho/sparql.rb

@@ -0,0 +1,332 @@
+module Pho
+
+  #Module providing a SPARQL client library, support for parsing SPARQL query responses into Ruby objects
+  #and other useful behaviour  
+  module Sparql
+
+    SPARQL_RESULTS_XML = "application/sparql-results+xml"
+    SPARQL_RESULTS_JSON = "application/sparql-results+json"
+    
+    #A simple SPARQL client that handles the basic HTTP traffic
+    class SparqlClient
+      
+        #URI of the endpoint
+        attr_reader :endpoint
+        #HTTPClient object
+        attr_reader :client
+        #Name of output parameter to use to control response format. If set then this parameter is added
+        #to the query string, rather than using Content Negotiation        
+        attr_accessor :output_parameter_name
+        attr_reader :graphs
+        attr_reader :named_graphs      
+        
+        #Configures whether the remote endpoint supports the RDF-in-JSON specification for serializing
+        #RDF graphs as JSON. Will default to false.
+        attr_accessor :supports_rdf_json
+        #Configures whether the remote endpoint supports SPARQL JSON Results format
+        #Will default to true.        
+        attr_accessor :supports_sparql_json
+                
+        #Initialize a client for a specific endpoint
+        #
+        # endpoint:: uri of the SPARQL endpoint
+        # client:: optionally, a reference to an existing HTTPClient object instance
+        def initialize(endpoint, client=HTTPClient.new() )
+         @endpoint = endpoint
+         @graphs = nil
+         @named_graphs = nil
+         @client = client
+         @output_parameter_name = nil
+         @supports_rdf_json = false
+         @supports_sparql_json = true
+        end 
+        
+        #Add a default graph. This will be added as a default graph in the request protocol
+        def add_default_graph(graph_uri)
+          if @graphs == nil
+             @graphs = []
+          end  
+          @graphs << graph_uri
+        end
+
+        #Add a named graph. This will be added as a named graph in the request protocol
+        def add_named_graph(graph_uri)
+          if @named_graphs == nil
+            @named_graphs = []
+          end  
+          @named_graphs << graph_uri
+        end
+                      
+        #Perform a sparql query
+        #
+        # sparql:: a valid SPARQL query
+        # format:: specific a request format. Usually a media-type, but may be a name for a type, if not using Conneg
+        # graphs:: an array of default graphs
+        # named_graphs:: an array of named graphs
+        def query(sparql, format=nil, graphs=nil, named_graphs=nil)
+          
+          params = {}
+          params["query"] = sparql
+          
+          if graphs != nil
+            params["default-graph-uri"] = graphs
+          elsif @graphs != nil
+            params["default-graph-uri"] = @graphs
+          end          
+
+          if named_graphs != nil
+            params["named-graph-uri"] = named_graphs
+          elsif @named_graphs != nil
+            params["named-graph-uri"] = @named_graphs
+          end
+          
+          headers = {}
+          if format != nil
+            
+            if @output_parameter_name != nil
+              params[@output_parameter_name] = format
+            else 
+              headers["Accept"] = format  
+            end
+            
+          end
+          
+          return @client.get( @endpoint, params, headers )
+        end
+        
+        #Perform a SPARQL DESCRIBE query.
+        #
+        # query:: the SPARQL query
+        # format:: the preferred response format
+        def describe(query, format="application/rdf+xml")
+          return query(query, format)
+        end
+
+        #DESCRIBE multiple resources in a single query. The provided array should contain
+        #the uris that are to be described
+        #
+        #This will generate a query like:
+        # DESCRIBE <http://www.example.org> <http://www.example.com> ...
+        #
+        # uris:: list of the uris to be described
+        # format:: the preferred response format. Default is RDF/XML
+        def multi_describe(uris, format="application/rdf+xml")
+          query = "DESCRIBE " + uris.map {|u| "<#{u}>" }.join(" ")
+          return query(query, format)
+        end
+              
+        #Perform a SPARQL CONSTRUCT query.
+        #
+        # query:: the SPARQL query
+        # format:: the preferred response format        
+        def construct(query, format="application/rdf+xml")
+          return query(query, format)
+        end
+        
+        #Perform a SPARQL ASK query.
+        #
+        # query:: the SPARQL query
+        # format:: the preferred response format    
+        def ask(query, format=Pho::Sparql::SPARQL_RESULTS_XML)
+          return query(query, format)
+        end
+        
+        #Perform a SPARQL SELECT query.
+        #
+        # query:: the SPARQL query
+        # format:: the preferred response format    
+        def select(query, format=Pho::Sparql::SPARQL_RESULTS_XML)
+          return query(query, format)
+        end
+        
+    end
+   
+    #Simple helper class for manipulating and executing SPARQL queries and manipulating the results
+    class SparqlHelper
+        VARIABLE_MATCHER = /(\?|\$)([a-zA-Z]+)/
+        
+        #Apply some initial bindings to parameters in a query
+        #
+        #The keys in the values hash are used to replace variables in a query
+        #The values are supplied as is, allowing them to be provided as URIs, or typed literals
+        #according to Turtle syntax.
+        #
+        #Any keys in the hash that are not in the query are ignored. Any variables not found
+        #in the hash remain unbound.
+        #
+        #  query:: the query whose initial bindings are to be set
+        #  values:: hash of query name to value
+        def SparqlHelper.apply_initial_bindings(query, bindings={})
+            copy = query.clone()
+            copy.gsub!(VARIABLE_MATCHER) do |pattern|
+              key = $2
+              if bindings.has_key?(key)
+                bindings[key].to_s
+              else
+                pattern
+              end              
+            end            
+            return copy  
+        end
+        
+        #Convert a SPARQL query result binding into a hash suitable for passing
+        #to the apply_initial_bindings method.
+        #
+        #The result param is assumed to be a Ruby hash that reflects the structure of 
+        #a binding in a SELECT query result (i.e. the result of parsing the <tt>application/sparql-results+json</tt> 
+        #format and extracting an specific result binding.
+        #
+        #The method is intended to be used to support cases where an initial select query is 
+        #performed to extract some variables that can later be plugged into a subsequent 
+        #query
+        #
+        # result:: hash conforming to structure of a <tt>binding</tt> in the SPARQL JSON format
+        def SparqlHelper.result_to_query_binding(result)
+          hash = {}
+          result.each_pair do |key, value|
+            if value["type"] == "uri"
+              hash[key] = "<#{value["value"]}>"
+            elsif (value["type"] == "literal" && !value.has_key?("datatype"))
+              hash[key] = "\"#{value["value"]}\""
+            elsif (value["type"] == "literal" && value.has_key?("datatype"))
+              hash[key] = "\"#{value["value"]}\"^^#{value["datatype"]}"             
+            else
+              #do nothing for bnodes
+            end
+          end
+          return hash
+        end
+        
+        #Convert Ruby hash structured according to SPARQL JSON format
+        #into an array of hashes by calling result_to_query_binding on each binding
+        #into the results.
+        #
+        #E.g:
+        # <tt>results = Pho::Sparql::SparqlHelper.select(query, sparql_client)</tt>
+        # <tt>bindings = Pho::Sparql::SparqlHelper.results_to_query_bindings(results)</tt>
+        #
+        #  results:: hash conforming to SPARQL SELECT structure
+        def SparqlHelper.results_to_query_bindings(results)
+          bindings = []
+          
+          results["results"]["bindings"].each do |result|
+            bindings << result_to_query_binding(result)
+          end
+          return bindings
+        end              
+        
+        #Perform a simple SELECT query on an endpoint.
+        #Will request the results using the SPARQL JSON results format, and parse the
+        #resulting JSON results. The result will therefore be a simple ruby hash of the results
+        #
+        #An error will be raised if the response is HTTP OK.
+        #
+        # query:: the SPARQL SELECT query
+        # sparql_client:: a configured SparqlClient object
+        def SparqlHelper.select(query, sparql_client)
+          #TODO: test whether endpoint supports json, and if not, switch to parsing XML
+          resp = sparql_client.select(query, Pho::Sparql::SPARQL_RESULTS_JSON)
+          if resp.status != 200
+            raise "Error performing sparql query: #{resp.status} #{resp.reason}\n#{resp.content}"
+          end
+          return JSON.parse( resp.content )
+        end
+    
+        #Performs an ASK query on an endpoint, returing a boolean true/false response
+        #
+        #Will request the results using the SPARQL JSON results format, parse the
+        #resulting JSON results, and extract the true/false response.
+        #
+        # query:: the SPARQL SELECT query
+        # sparql_client:: a configured SparqlClient object
+        def SparqlHelper.ask(query, sparql_client)
+          json = SparqlHelper.select(query, sparql_client)
+          return json["boolean"] == "true"
+        end
+    
+        #Perform a simple SELECT query on an endpoint and return a simple array of values
+        #
+        #Will request the results using the SPARQL JSON results format, and parse the
+        #resulting JSON results. The assumption is that the SELECT query contains a single "column" 
+        #of values, which will be returned as an array 
+        #
+        #Note this will lose any type information, only the value of the bindings are returned 
+        #
+        # query:: the SPARQL SELECT query
+        # sparql_client:: a configured SparqlClient object
+        def SparqlHelper.select_values(query, sparql_client)
+           results = SparqlHelper.select(query, sparql_client)
+           v = results["head"]["vars"][0];
+           values = [];
+           results["results"]["bindings"].each do |binding|
+             values << binding[v]["value"]
+           end
+           return values           
+        end
+
+        #Perform a simple SELECT query on an endpoint and return a single result
+        #
+        #Will request the results using the SPARQL JSON results format, and parse the
+        #resulting JSON results. The assumption is that the SELECT query returns a single
+        #value (i.e single variable, with single binding)
+        #
+        #Note this will lose any type information, only the value of the binding is returned 
+        #
+        # query:: the SPARQL SELECT query
+        # sparql_client:: a configured SparqlClient object                
+        def SparqlHelper.select_single_value(query, sparql_client)
+          results = SparqlHelper.select(query, sparql_client)
+          v = results["head"]["vars"][0];
+          return results["results"]["bindings"][0][v]["value"]           
+        end
+        
+        #Perform a SPARQL CONSTRUCT query against an endpoint, requesting the results in JSON 
+        #
+        #Will request the results as application/json (with the expectation that it returns RDF_JSON), 
+        #and parses the resulting JSON document.
+        #
+        # query:: the SPARQL SELECT query
+        # sparql_client:: a configured SparqlClient object                        
+        def SparqlHelper.construct_to_resource_hash(query, sparql_client)
+          #TODO: test whether endpoint supports json, and if not, switch to parsing XML
+          resp = sparql_client.construct(query, "application/json")
+          if resp.status != 200
+            raise "Error performing sparql query: #{resp.status} #{resp.reason}\n#{resp.content}"
+          end
+          return Pho::ResourceHash::Converter.parse_json( resp.content )          
+        end
+
+        #Perform a SPARQL DESCRIBE query against an endpoint, requesting the results in JSON 
+        #
+        #Will request the results as application/json (with the expectation that it returns RDF_JSON), 
+        #and parses the resulting JSON document.
+        #
+        # query:: the SPARQL SELECT query
+        # sparql_client:: a configured SparqlClient object                                
+        def SparqlHelper.describe_to_resource_hash(query, sparql_client)
+          #TODO: test whether endpoint supports json, and if not, switch to parsing XML
+          resp = sparql_client.describe(query, "application/json")
+          if resp.status != 200
+            raise "Error performing sparql query: #{resp.status} #{resp.reason}\n#{resp.content}"
+          end
+          return Pho::ResourceHash::Converter.parse_json( resp.content )          
+        end
+
+        #DESCRIBE multiple resources in a single SPARQL request
+        #
+        # uris:: an array of URIs
+        # sparql_client:: a configured SparqlClient objec                        
+        def SparqlHelper.multi_describe(uris, sparql_client)
+          #TODO: test whether endpoint supports json, and if not, switch to parsing XML
+          resp = sparql_client.multi_describe(uris, "application/json")
+          if resp.status != 200
+            raise "Error performing sparql query: #{resp.status} #{resp.reason}\n#{resp.content}"
+          end
+          return Pho::ResourceHash::Converter.parse_json( resp.content )                    
+        end
+                
+    end     
+        
+  end
+  
+end