pho 0.5 → 0.6

data/lib/pho/file_manager.rb

@@ -6,44 +6,40 @@ module Pho
 
     class FileManager < AbstractFileManager
    
-      def initialize(store, dir, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
+      attr_reader :base
+      
+      def initialize(store, dir, base = nil, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
         super(store, dir, ok_suffix, fail_suffix, sleep)
+        @base = base
       end
-                
+
       #List files being managed, i.e. everything not .ok or .fail
-      def list()          
+      def list(recursive=false)          
           files = []
-          Dir.glob( File.join(@dir, "*") ) do |file|
-            if File.extname(file) != ".#{@ok_suffix}" && File.extname(file) != ".#{@fail_suffix}" 
+          if recursive
+            pattern = "**/*"
+          else
+            pattern = "*"
+          end
+          Dir.glob( File.join(@dir, pattern) ) do |file|
+            if File.extname(file) != ".#{@ok_suffix}" && File.extname(file) != ".#{@fail_suffix}" && !File.directory?(file)
               files << file
             end
           end  
           return files
       end
-                      
-      #List any new files in the directory
-      def new_files()
-        newfiles = Array.new
-        Dir.glob( File.join(@dir, "*") ) do |file|
-          
-          if File.extname(file) != ".#{@ok_suffix}" && File.extname(file) != ".#{@fail_suffix}" 
-            ok_file = get_ok_file_for(file)
-            fail_file = get_fail_file_for(file)
-            if !( File.exists?(ok_file) or File.exists?(fail_file) )
-              newfiles << file          
-            end
-            
-          end
-          
-        end
-        return newfiles
-      end
-
-       
+           
+      def FileManager.name_for_file(dir, file, base=nil)
+        uri = file.path.gsub(dir, "")
+        uri = "#{base}#{uri}" if base != nil
+        return uri
+      end                  
+      
       protected
   
       def store_file(file, filename)
-        response = @store.upload_item(file, MIME::Types.type_for(filename)[0].to_s )
+        uri = FileManager.name_for_file(@dir, file, @base)
+        response = @store.upload_item(file, MIME::Types.type_for(filename)[0].to_s, uri )
         if (response.status < 300 )
           File.open(get_ok_file_for(filename), "w") do |file|
             file.print( "OK" )            

data/lib/pho/rdf.rb

@@ -0,0 +1,74 @@
+module Pho
+
+  #Module containing general RDF utilities and classes
+  #
+  #Dependent on the redland ruby bindings
+  module RDF
+    
+    begin
+      require 'rdf/redland'
+    rescue LoadError
+      $stderr.puts "WARNING: Unable to load redland-ruby bindings. Some RDF utils will be unavailable"
+    end
+    
+    #General RDF parsing utilities.
+    #
+    #Currently a convenience wrapper around the Redland Ruby bindings 
+    class Parser
+      
+      #Parse a file containing ntriples into RDF/XML. Returns a string
+      #
+      # file:: File object
+      # base_uri:: optional base uri to be used when parsing. If not set, then uris are resolved
+      # relative to the File
+      def Parser.parse_ntriples(file, base_uri=nil)
+        model = Redland::Model.new()
+        parser = Redland::Parser.new("ntriples", "")
+        uri = "file:#{file.path}"
+        base_uri = uri unless base_uri
+        parser.parse_into_model(model, uri, base_uri)
+        serializer = Redland::Serializer.new( "rdfxml", "application/rdf+xml" )
+        data = serializer.model_to_string(Redland::Uri.new(base_uri), model)
+        return data                        
+      end
+      
+      #Parse a string containing ntriples into RDF/XML. Returns a string
+      #
+      # string:: the string containing the data
+      # base_uri:: base uri for parsing the data
+      def Parser.parse_ntriples_from_string(string, base_uri)
+        model = Redland::Model.new()
+        parser = Redland::Parser.new("ntriples", "")
+        parser.parse_string_into_model(model, string, Redland::Uri.new(base_uri))
+        serializer = Redland::Serializer.new( "rdfxml", "application/rdf+xml" )
+        data = serializer.model_to_string(Redland::Uri.new(base_uri), model)
+        return data                                
+      end
+
+      #Convenience method to parse an ntriples file and store it a Platform store
+      #
+      # file:: the file to parse
+      # store:: the store to receive the data
+      # base_uri:: base uri against which the data is parsed
+      # graph_name:: uri of graph in store
+      def Parser.store_ntriples(file, store, base_uri=nil, graph_name=nil)
+         data = Parser.parse_ntriples(file, base_uri)
+         return store.store_data(data, graph_name)       
+      end
+
+      #Convenience method to parse an ntriples string and store it a Platform store
+      #
+      # string:: the data to parse
+      # store:: the store to receive the data
+      # base_uri:: base uri against which the data is parsed
+      # graph_name:: uri of graph in store
+      def Parser.store_ntriples_from_string(string, store, base_uri, graph_name=nil)
+         data = Parser.parse_ntriples_from_string(string, base_uri)
+         return store.store_data(data, graph_name)       
+      end
+      
+    end
+
+  end
+
+end

data/lib/pho/rdf_collection.rb

@@ -14,26 +14,11 @@ module Pho
           super(store, dir, ok_suffix, fail_suffix, sleep)
           @rdf_suffix = rdf_suffix
         end
-    
-                
+                    
         #List files being managed
-        def list()
+        def list(recursive=false)
             return Dir.glob( File.join(@dir, "*.#{@rdf_suffix}") )  
         end
-                        
-        #List any new files in the directory
-        def new_files()
-          newfiles = Array.new
-          Dir.glob( File.join(@dir, "*.#{@rdf_suffix}") ) do |file|
-            ok_file = get_ok_file_for(file)
-            fail_file = get_fail_file_for(file)
-            if !( File.exists?(ok_file) or File.exists?(fail_file) )
-              newfiles << file          
-            end
-          end
-          return newfiles
-        end
-
          
         protected
     

data/lib/pho/sparql.rb

@@ -7,6 +7,90 @@ module Pho
     SPARQL_RESULTS_XML = "application/sparql-results+xml"
     SPARQL_RESULTS_JSON = "application/sparql-results+json"
     
+    #Includes all statements along both in-bound and out-bound arc paths
+    #
+    #See http://n2.talis.com/wiki/Bounded_Descriptions_in_RDF   
+    SYMMETRIC_BOUNDED_DESCRIPTION = <<-EOL
+    CONSTRUCT {?uri ?p ?o . ?s ?p2 ?uri .} WHERE { {?uri ?p ?o .} UNION {?s ?p2 ?uri .} }
+    EOL
+    
+    #Similar to Concise Bounded Description but includes labels for referenced resources
+    #
+    #See http://n2.talis.com/wiki/Bounded_Descriptions_in_RDF    
+    LABELLED_BOUNDED_DESCRIPTION = <<-EOL
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
+    CONSTRUCT {
+       ?uri ?p ?o . 
+       ?o rdfs:label ?label . 
+       ?o rdfs:comment ?comment . 
+       ?o <http://www.w3.org/2004/02/skos/core#prefLabel> ?plabel . 
+       ?o rdfs:seeAlso ?seealso.
+    } WHERE {
+      ?uri ?p ?o . 
+      OPTIONAL { 
+        ?o rdfs:label ?label .
+      } 
+      OPTIONAL {
+        ?o <http://www.w3.org/2004/02/skos/core#prefLabel> ?plabel . 
+      } 
+      OPTIONAL {
+        ?o rdfs:comment ?comment . 
+      } 
+      OPTIONAL { 
+        ?o rdfs:seeAlso ?seealso.
+      }
+    }    
+    EOL
+
+    #Derived from both the Symmetric and Labelled Bounded Descriptions. Includes all in-bound
+    #and out-bound arc paths, with labels for any referenced resources.
+    #
+    #See http://n2.talis.com/wiki/Bounded_Descriptions_in_RDF    
+    SYMMETRIC_LABELLED_BOUNDED_DESCRIPTION = <<-EOL
+    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
+    CONSTRUCT {
+      ?uri ?p ?o . 
+      ?o rdfs:label ?label . 
+      ?o rdfs:comment ?comment . 
+      ?o rdfs:seeAlso ?seealso. 
+      ?s ?p2 ?uri . 
+      ?s rdfs:label ?label . 
+      ?s rdfs:comment ?comment . 
+      ?s rdfs:seeAlso ?seealso.
+    } WHERE { 
+      { ?uri ?p ?o . 
+        OPTIONAL { 
+          ?o rdfs:label ?label .
+        } 
+        OPTIONAL {
+          ?o rdfs:comment ?comment .
+        } 
+        OPTIONAL {
+          ?o rdfs:seeAlso ?seealso.
+        } 
+      } 
+      UNION {
+        ?s ?p2 ?uri . 
+        OPTIONAL {
+          ?s rdfs:label ?label .
+        } 
+        OPTIONAL {
+          ?s rdfs:comment ?comment .
+        } 
+        OPTIONAL {
+          ?s rdfs:seeAlso ?seealso.
+        } 
+      } 
+    }    
+    EOL
+
+    DESCRIPTIONS = {
+      :cbd => "DESCRIBE ?uri",
+      :scbd => SYMMETRIC_BOUNDED_DESCRIPTION,
+      :lcbd => LABELLED_BOUNDED_DESCRIPTION,
+      :slcbd => SYMMETRIC_LABELLED_BOUNDED_DESCRIPTION
+    }      
+    
     #A simple SPARQL client that handles the basic HTTP traffic
     class SparqlClient
       
@@ -29,8 +113,8 @@ module Pho
                 
         #Initialize a client for a specific endpoint
         #
-        # endpoint:: uri of the SPARQL endpoint
-        # client:: optionally, a reference to an existing HTTPClient object instance
+        #endpoint:: uri of the SPARQL endpoint
+        #client:: optionally, a reference to an existing HTTPClient object instance
         def initialize(endpoint, client=HTTPClient.new() )
          @endpoint = endpoint
          @graphs = nil
@@ -59,10 +143,10 @@ module Pho
                       
         #Perform a sparql query
         #
-        # sparql:: a valid SPARQL query
-        # format:: specific a request format. Usually a media-type, but may be a name for a type, if not using Conneg
-        # graphs:: an array of default graphs
-        # named_graphs:: an array of named graphs
+        #sparql:: a valid SPARQL query
+        #format:: specific a request format. Usually a media-type, but may be a name for a type, if not using Conneg
+        #graphs:: an array of default graphs
+        #named_graphs:: an array of named graphs
         def query(sparql, format=nil, graphs=nil, named_graphs=nil)
           
           params = {}
@@ -94,10 +178,24 @@ module Pho
           return @client.get( @endpoint, params, headers )
         end
         
+        #Describe a uri, optionally specifying a form of bounded description
+        #
+        #uri:: the uri to describe
+        #format:: mimetype for results
+        #type:: symbol indicating type of description, i.e. +:cbd+, +:scbd+, +:lcbd+, or +:slcbd+
+        def describe_uri(uri, format="application/rdf+xml", type=:cbd)
+          template = Pho::Sparql::DESCRIPTIONS[type]
+          if template == nil
+            raise "Unknown description type"
+          end
+          query = Pho::Sparql::SparqlHelper.apply_initial_bindings(template, {"uri" => "<#{uri}>"} )
+          return describe(query, format)
+        end
+        
         #Perform a SPARQL DESCRIBE query.
         #
-        # query:: the SPARQL query
-        # format:: the preferred response format
+        #query:: the SPARQL query
+        #format:: the preferred response format
         def describe(query, format="application/rdf+xml")
           return query(query, format)
         end
@@ -108,8 +206,8 @@ module Pho
         #This will generate a query like:
         # DESCRIBE <http://www.example.org> <http://www.example.com> ...
         #
-        # uris:: list of the uris to be described
-        # format:: the preferred response format. Default is RDF/XML
+        #uris:: list of the uris to be described
+        #format:: the preferred response format. Default is RDF/XML
         def multi_describe(uris, format="application/rdf+xml")
           query = "DESCRIBE " + uris.map {|u| "<#{u}>" }.join(" ")
           return query(query, format)
@@ -117,24 +215,24 @@ module Pho
               
         #Perform a SPARQL CONSTRUCT query.
         #
-        # query:: the SPARQL query
-        # format:: the preferred response format        
+        #query:: the SPARQL query
+        #format:: the preferred response format        
         def construct(query, format="application/rdf+xml")
           return query(query, format)
         end
         
         #Perform a SPARQL ASK query.
         #
-        # query:: the SPARQL query
-        # format:: the preferred response format    
+        #query:: the SPARQL query
+        #format:: the preferred response format    
         def ask(query, format=Pho::Sparql::SPARQL_RESULTS_XML)
           return query(query, format)
         end
         
         #Perform a SPARQL SELECT query.
         #
-        # query:: the SPARQL query
-        # format:: the preferred response format    
+        #query:: the SPARQL query
+        #format:: the preferred response format    
         def select(query, format=Pho::Sparql::SPARQL_RESULTS_XML)
           return query(query, format)
         end
@@ -154,8 +252,8 @@ module Pho
         #Any keys in the hash that are not in the query are ignored. Any variables not found
         #in the hash remain unbound.
         #
-        #  query:: the query whose initial bindings are to be set
-        #  values:: hash of query name to value
+        #query:: the query whose initial bindings are to be set
+        #values:: hash of query name to value
         def SparqlHelper.apply_initial_bindings(query, bindings={})
             copy = query.clone()
             copy.gsub!(VARIABLE_MATCHER) do |pattern|
@@ -180,7 +278,7 @@ module Pho
         #performed to extract some variables that can later be plugged into a subsequent 
         #query
         #
-        # result:: hash conforming to structure of a <tt>binding</tt> in the SPARQL JSON format
+        #result:: hash conforming to structure of a <tt>binding</tt> in the SPARQL JSON format
         def SparqlHelper.result_to_query_binding(result)
           hash = {}
           result.each_pair do |key, value|
@@ -202,10 +300,10 @@ module Pho
         #into the results.
         #
         #E.g:
-        # <tt>results = Pho::Sparql::SparqlHelper.select(query, sparql_client)</tt>
-        # <tt>bindings = Pho::Sparql::SparqlHelper.results_to_query_bindings(results)</tt>
+        #<tt>results = Pho::Sparql::SparqlHelper.select(query, sparql_client)</tt>
+        #<tt>bindings = Pho::Sparql::SparqlHelper.results_to_query_bindings(results)</tt>
         #
-        #  results:: hash conforming to SPARQL SELECT structure
+        #results:: hash conforming to SPARQL SELECT structure
         def SparqlHelper.results_to_query_bindings(results)
           bindings = []
           
@@ -221,8 +319,8 @@ module Pho
         #
         #An error will be raised if the response is HTTP OK.
         #
-        # query:: the SPARQL SELECT query
-        # sparql_client:: a configured SparqlClient object
+        #query:: the SPARQL SELECT query
+        #sparql_client:: a configured SparqlClient object
         def SparqlHelper.select(query, sparql_client)
           #TODO: test whether endpoint supports json, and if not, switch to parsing XML
           resp = sparql_client.select(query, Pho::Sparql::SPARQL_RESULTS_JSON)
@@ -237,8 +335,8 @@ module Pho
         #Will request the results using the SPARQL JSON results format, parse the
         #resulting JSON results, and extract the true/false response.
         #
-        # query:: the SPARQL SELECT query
-        # sparql_client:: a configured SparqlClient object
+        #query:: the SPARQL SELECT query
+        #sparql_client:: a configured SparqlClient object
         def SparqlHelper.ask(query, sparql_client)
           json = SparqlHelper.select(query, sparql_client)
           return json["boolean"] == "true"
@@ -252,8 +350,8 @@ module Pho
         #
         #Note this will lose any type information, only the value of the bindings are returned 
         #
-        # query:: the SPARQL SELECT query
-        # sparql_client:: a configured SparqlClient object
+        #query:: the SPARQL SELECT query
+        #sparql_client:: a configured SparqlClient object
         def SparqlHelper.select_values(query, sparql_client)
            results = SparqlHelper.select(query, sparql_client)
            v = results["head"]["vars"][0];
@@ -272,8 +370,8 @@ module Pho
         #
         #Note this will lose any type information, only the value of the binding is returned 
         #
-        # query:: the SPARQL SELECT query
-        # sparql_client:: a configured SparqlClient object                
+        #query:: the SPARQL SELECT query
+        #sparql_client:: a configured SparqlClient object                
         def SparqlHelper.select_single_value(query, sparql_client)
           results = SparqlHelper.select(query, sparql_client)
           v = results["head"]["vars"][0];
@@ -285,8 +383,8 @@ module Pho
         #Will request the results as application/json (with the expectation that it returns RDF_JSON), 
         #and parses the resulting JSON document.
         #
-        # query:: the SPARQL SELECT query
-        # sparql_client:: a configured SparqlClient object                        
+        #query:: the SPARQL SELECT query
+        #sparql_client:: a configured SparqlClient object                        
         def SparqlHelper.construct_to_resource_hash(query, sparql_client)
           #TODO: test whether endpoint supports json, and if not, switch to parsing XML
           resp = sparql_client.construct(query, "application/json")
@@ -301,8 +399,8 @@ module Pho
         #Will request the results as application/json (with the expectation that it returns RDF_JSON), 
         #and parses the resulting JSON document.
         #
-        # query:: the SPARQL SELECT query
-        # sparql_client:: a configured SparqlClient object                                
+        #query:: the SPARQL SELECT query
+        #sparql_client:: a configured SparqlClient object                                
         def SparqlHelper.describe_to_resource_hash(query, sparql_client)
           #TODO: test whether endpoint supports json, and if not, switch to parsing XML
           resp = sparql_client.describe(query, "application/json")
@@ -311,11 +409,11 @@ module Pho
           end
           return Pho::ResourceHash::Converter.parse_json( resp.content )          
         end
-
+        
         #DESCRIBE multiple resources in a single SPARQL request
         #
-        # uris:: an array of URIs
-        # sparql_client:: a configured SparqlClient objec                        
+        #uris:: an array of URIs
+        #sparql_client:: a configured SparqlClient objec                        
         def SparqlHelper.multi_describe(uris, sparql_client)
           #TODO: test whether endpoint supports json, and if not, switch to parsing XML
           resp = sparql_client.multi_describe(uris, "application/json")
@@ -324,7 +422,21 @@ module Pho
           end
           return Pho::ResourceHash::Converter.parse_json( resp.content )                    
         end
-                
+   
+        #Describe a single URI using one of several forms of Bounded Description
+        #See SparqlClient.describe_uri
+        #
+        #uri:: resource to describe
+        #sparql_client:: configured SPARQL client
+        #type:: form of bounded description to generate
+        def SparqlHelper.describe_uri(uri, sparql_client, type=:cbd)
+          #TODO: test whether endpoint supports json, and if not, switch to parsing XML
+          resp = sparql_client.describe_uri(uri, "application/json", type)
+          if resp.status != 200
+            raise "Error performing sparql query: #{resp.status} #{resp.reason}\n#{resp.content}"
+          end
+          return Pho::ResourceHash::Converter.parse_json( resp.content )                              
+        end
     end     
         
   end