pho 0.5 → 0.6

data/examples/sinatra/viewer.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'pho'
+require 'sinatra'
+
+#SETUP
+store = Pho::Store.new("http://api.talis.com/stores/space")
+mime :rdf, "application/rdf+xml"
+
+#ROUTES
+
+#E.g. http://127.0.0.1:4567/?url=http://nasa.dataincubator.org/spacecraft/1969-059A
+get "/*" do  
+  sparql_client = store.sparql_client()
+  resp = sparql_client.describe_uri(params[:url], "application/rdf+xml", :lcbd)
+  
+  if resp.status != 200
+    halt resp.status, resp.message
+  end
+  content_type :rdf
+  resp.content

data/lib/pho.rb

@@ -16,11 +16,14 @@ require 'pho/file_management'
 require 'pho/rdf_collection'
 require 'pho/file_manager'
 require 'pho/resource_hash'
-require 'pho/converter'
 require 'pho/changeset_builder'
 require 'pho/changeset'
 require 'pho/sparql'
 require 'pho/enrichment'
+require 'pho/command_line'
+#Following depend on redland
+require 'pho/converter'
+require 'pho/rdf'
 
 if RUBY_VERSION < "1.8.7"
   class String

data/lib/pho/changeset.rb

@@ -320,7 +320,23 @@ module Pho
 
     #end changeset   
     end
-       
+   
+    #Utility methods for making changes to graphs via Changesets
+    class ChangesetHelper
+    
+      def ChangesetHelper.update_literal(store, subject, predicate, old_value, new_value, old_lang=nil, old_datatype=nil, 
+          new_lang=nil, new_datatype=nil, creator_name=nil, change_reason=nil, versioned=false)
+        cs = Changeset.new(subject, creator_name, change_reason)
+        old = Statement.create_literal(subject, predicate, old_value, old_lang, old_datatype)
+        new = Statement.create_literal(subject, predicate, new_value, new_lang, new_datatype)
+        cs.add_removal(old)
+        cs.add_addition(new)
+        return cs.submit(store, versioned)
+      end
+      
+      
+    end    
+    
   #end module                   
   end
 

data/lib/pho/command_line.rb

@@ -0,0 +1,166 @@
+module Pho
+
+  #Class that implements the command-line behaviour
+  class CommandLine
+
+    def initialize(store, opts)
+       @store = store      
+       @opts = opts
+    end
+    
+    def status()
+      status = Pho::Status.read_from_store(@store)
+      puts "Store Status:\nReadable: #{status.readable?}\nWritable: #{status.writeable?}"      
+    end
+    
+    def backup()
+      puts "Submitting Snapshot Job"
+      resp = Pho::Jobs.submit_snapshot(@store, "Reindex", Time.now)
+      puts "Monitoring Job"
+      job = Pho::Jobs.wait_for_submitted(resp, @store) do |job, message, time|
+         puts "#{time} #{message}"
+      end 
+      puts "Job Completed"         
+      snapshot = Pho::Snapshot.read_from_store(@store)
+      puts "Retrieving #{snapshot.url}"
+      dir = @opts["dir"] || Dir.tmpdir
+      snapshot.backup(@store, dir)
+      puts "Download complete. MD5 OK."      
+    end
+    
+    def snapshot()
+      puts "Submitting Snapshot Job"
+      resp = Pho::Jobs.submit_snapshot(@store, "Reindex", Time.now)
+      puts "Monitoring Job"
+      job = Pho::Jobs.wait_for_submitted(resp, @store) do |job, message, time|
+        puts "#{time} #{message}"
+      end 
+      puts "Snapshot Completed"      
+    end
+    
+    def reindex()
+      puts "Submitting Reindex Job"
+      resp = Pho::Jobs.submit_reindex(@store, "Reindex", Time.now)
+      if resp.status == 201
+          puts "Monitoring Reindex Job: #{resp.header["Location"].first}"
+      end
+      job = Pho::Jobs.wait_for_submitted(resp, @store) do |job, message, time|
+          puts "#{time} #{message}"
+      end 
+      puts "Reindex Completed"      
+    end
+    
+    def reset()
+      puts "Resetting Store"
+      resp = Pho::Jobs.submit_reset(@store, "Reset", Time.now)
+      puts "Monitoring Reset Job:"
+      job = Pho::Jobs.wait_for_submitted(resp, @store) do |job, message, time|
+        puts "#{time} #{message}"
+      end 
+      puts "Reset Completed"      
+    end
+    
+    def restore()
+      url = @opts["url"]     
+      if url == nil
+        puts "Restore from latest snapshot"
+        snapshot = Pho::Snapshot.read_from_store(@store)
+        url = snapshot.url
+      end
+      puts "Restoring from #{url}"    
+      resp = Pho::Jobs.submit_restore(@store, url, "Reset", Time.now)
+      job = Pho::Jobs.wait_for_submitted(resp, @store) do |job, message, time|
+        puts "#{time} #{message}"
+      end 
+      puts "Restore Completed"      
+    end
+    
+    def describe()
+      resp = @store.describe( @opts["url"] )
+      if resp.status == 200
+        puts resp.content
+      else
+        puts "Error: #{resp.status} #{resp.reason}"
+        puts resp.content
+      end    
+    end  
+
+    def sparql()
+      query = File.new( @opts["file"] ).read()
+      resp = @store.sparql(query)
+      if resp.status == 200
+        puts resp.content
+      else
+        puts "Error: #{resp.status} #{resp.reason}"
+        puts resp.content          
+      end      
+    end
+
+    def store()
+      resp = nil
+      if @opts["url"]
+        puts "Storing remote data: #{@opts["url"]}"
+        resp = @store.store_url( @opts["url"] ) 
+      elsif @opts["file"]
+        puts "Storing local file: #{@opts["file"]}"
+        f = File.new( @opts["file"] )
+        if File.extname(f) == ".nt"
+          Pho::RDF::Parser.store_ntriples(f, @store)
+        else
+          resp = @store.store_file( f )  
+        end          
+      elsif @opts["dir"]
+        puts "Storing contents of directory: #{@opts["dir"]}"
+        collection = Pho::RDFCollection.new(@store, @opts["dir"])         
+        collection.store()          
+        puts collection.summary()
+      else     
+        #noop
+      end 
+    
+      if resp != nil 
+        if resp.status == 204
+          puts "Complete"
+        else
+          puts "Error: #{resp.status} #{resp.reason}"
+          puts resp.content
+        end         
+      end         
+    end     
+         
+    def upload()
+      resp = nil
+      if @opts["file"]
+        f = File.new( @opts["file"] )
+        uri = File.basename( @opts["file"] )
+        uri = "#{opts["base"]}/#{uri}" if @opts["base"]            
+        mime = MIME::Types.type_for( @opts["file"] )[0].to_s
+        puts "Uploading file: #{ @opts["file"] } to /items/#{ uri } as #{mime}"
+        resp = @store.upload_item( f , mime , uri )
+      elsif @opts["dir"]
+        puts "Uploading contents of directory: #{@opts["dir"]}"
+        collection = Pho::FileManagement::FileManager.new(@store, @opts["dir"], @opts["base"])
+        if opts["traverse"]
+          collection.store(:traverse)          
+          puts collection.summary(:traverse)
+        else
+          collection.store()          
+          puts collection.summary()            
+        end         
+      else     
+        #noop
+      end 
+    
+      if resp != nil
+        if resp.status == 204
+          puts "Complete"
+        else
+          puts "Error: #{resp.status} #{resp.reason}"
+          puts resp.content
+        end         
+      end                
+    end
+
+  end
+  
+end

data/lib/pho/converter.rb

@@ -1,12 +1,17 @@
 module Pho
   
+  #Resource Hashes
+  #
+  #Dependent on the redland ruby bindings  
   module ResourceHash
 
-    #TODO reading from streams
-    #TODO convert into other formats
     #TODO wrap Redland exceptions. Parser/Serializer contruction as well as parsing errors
     
-    require 'rdf/redland'
+    begin
+      require 'rdf/redland'
+    rescue LoadError
+      $stderr.puts "WARNING: Unable to load redland-ruby bindings. Changeset support unavailable"
+    end
     
     #Class for converting to and from resource hashes 
     class Converter

data/lib/pho/enrichment.rb

@@ -4,14 +4,15 @@ module Pho
   #external SPARQL endpoints and services
   module Enrichment
     
-    class ResourceEnricher
+    class StoreEnricher
       
       attr_reader :store
       attr_reader :sparql_client
       
-      # Constructor
-      #  store:: the store containing resource(s) to enrich
-      #  sparql_client:: SparqlClient object to be used as client for retrieving enrichments
+      #Constructor
+      #
+      #store:: the store containing resource(s) to enrich
+      #sparql_client:: SparqlClient object to be used as client for retrieving enrichments
       def initialize(store, sparql_client)
           @store = store
           @sparql_client = sparql_client  
@@ -22,56 +23,120 @@ module Pho
       #For the common case where a store needs to be enriched against itself by inferring new 
       #data from old using a CONSTRUCT query
       #
-      # store:: the store to be updated
-      # query:: the SPARQL CONSTRUCT query to be used to infer the new data
-      def ResourceEnricher.infer(store, query)
-        enricher = ResourceEnricher.new( store, store.sparql_client() )
-        return enricher.merge( query )
+      #store:: the store to be updated
+      #query:: the SPARQL CONSTRUCT query to be used to infer the new data
+      def StoreEnricher.infer(store, query, &block)
+        enricher = StoreEnricher.new( store, store.sparql_client() )
+        return enricher.merge( query, &block )
       end 
             
       #Execute the provided query against the configured SPARQL endpoint and store the results in 
       #the Platform Store
       #
-      # query:: the SPARQL CONSTRUCT or DESCRIBE query to execute
+      #query:: the SPARQL CONSTRUCT or DESCRIBE query to execute
       def merge(query)
         resp = @sparql_client.query( query, "application/rdf+xml" )
         if resp.status != 200
            raise "Unable to execute query. Response: #{resp.status} #{resp.reason} #{resp.content}"  
         end
-        resp = @store.store_data( resp.content )
+        data = resp.content
+        resp = @store.store_data( data )
+        if block_given?
+          yield resp, data
+        end
+        
         return resp        
       end
       
       #TODO: optimize POSTs back to the Platform, to deal with large number of resources, e.g. batching
       
-      # Enrich a store with data extracted from a SPARQL endpoint.
+      #Enrich a store with data extracted from a SPARQL endpoint.
+      #
+      #The locator_query is used to find resources in the Platform Store that should be enriched. The query
+      #should be a SPARQL SELECT query that returns the data to be used to parameterize the enrichment_query.
+      #
+      #For each query result, the enrichment_query will be submitted to the configured SPARQL endpoint, after 
+      #first interpolating the string, providing the query result bindings as parameters. (See SparqlHelper.apply_initial_bindings 
+      #and SparqlHelper.results_to_query_bindings 
+      #
+      #If successful, the result of each enrichment query will then be pushed back into the Platform Store by 
+      #posting the results of the query to the metabox. Enrichment queries should therefore be CONSTRUCT or 
+      #DESCRIBE queries. The SPARQL protocol request will be sent with an Accept header of application/rdf+xml
+      #
+      #The method supports a callback object that can be provided as an optional parameter to the query. If provided then 
+      #then object should respond to either or both of the following methods:
+      #  pre_process(rdf_xml)
+      #  post_process(resp, rdf_xml)
+      #The first of these is invoked after each enrichment query has been executed on the configured SPARQL endpoint. It is 
+      #intended to support additional filtering or annotation. If the +pre_process+ method returns nil, then no data will be written 
+      #to the store, otherwise the return value is substituted instead of the original value.
+      #
+      #The second callback method, +post_process+ is called after data has been written to the store and provides access to the 
+      #response from the store, and the RDF/XML data that had been attempted to be stored. As the request may have been un-successful, 
+      #code should check the status on the HTTPMessage parameter.
       #
-      # The locator_query is used to find resources in the Platform Store that should be enriched. The query
-      # should be a SPARQL SELECT query that returns the data to be used to parameterize the enrichment_query.
+      #  class MyCallback
+      #     def pre_process(rdf)
+      #       if !should_store?(rdf)
+      #         return nil  
+      #       end 
+      #       return rdf
+      #     end
+      #     def post_process(resp, rdf)
+      #       puts "Store returned #{resp.status} when storing: #{rdf}"
+      #     end
+      #  end
+      #  callback = MyCallback.new()
+      #  enricher.enrich("SELECT ?item WHERE { ?item a ex:Class } LIMIT 10", "DESCRIBE ?item", callback)
       #
-      # For each query result, the enrichment_query will be submitted to the configured SPARQL endpoint, after 
-      # first interpolating the string, providing the query result bindings as parameters. (See SparqlHelper.apply_initial_bindings 
-      # and SparqlHelper.results_to_query_bindings 
+      #The callback support is primarily intended to support filtering and notification of activities. For simple logging purposes, the 
+      #method also supports a block parameter. This is invoked after each enrichment query, and each store response. The block can 
+      #receive two values: the first is a symbol (either +:query+ or +:store+) indicating the source of the response, and the response
+      #object itself. E.g:
       #
-      # If successful, the result of each enrichment query will then be pushed back into the Platform Store by 
-      # posting the results of the query to the metabox. Enrichment queries should therefore be CONSTRUCT or 
-      # DESCRIBE queries. The SPARQL protocol request will be sent with an Accept header of application/rdf+xml
+      #  enricher.enrich("SELECT ?item WHERE { ?item a ex:Class } LIMIT 10", "DESCRIBE ?item") do |source, resp|
+      #    if source == :query
+      #      puts "Enrichment query returned #{resp.status}"
+      #    else
+      #      puts "Store returned #{resp.status} when storing data"
+      #    end 
+      #  end
       #
-      #  locator_query:: query to locate resources to be enriched
-      #  enrichment_query:: query to be used to enrich the resource
-      def enrich(locator_query, enrichment_query)
+      #locator_query:: query to locate resources to be enriched
+      #enrichment_query:: query to be used to enrich the resource
+      #callback:: optional callback object
+      def enrich(locator_query, enrichment_query, callback=nil)
           results = Pho::Sparql::SparqlHelper.select(locator_query, @store.sparql_client() )
           bindings = Pho::Sparql::SparqlHelper.results_to_query_bindings(results)
           bindings.each do |binding|
             bound_query = Pho::Sparql::SparqlHelper.apply_initial_bindings(enrichment_query, binding)
 
-            result = @sparql_client.query(bound_query, "application/rdf+xml")
+            #TODO counting numbers of requests and responses?
+            query_response = @sparql_client.query(bound_query, "application/rdf+xml")           
             
-            #TODO error handling and callbacks            
-            if result.status == 200
-              resp = @store.store_data( result.content )  
-            end        
+            if block_given?
+              yield :query, query_response
+            end
             
+            if query_response.status == 200                            
+              result = query_response.content              
+              if callback != nil && callback.respond_to?(:pre_process)
+                result = callback.pre_process(result)
+              end
+                                 
+              if result != nil                
+                store_response = @store.store_data( result )                
+                if callback != nil && callback.respond_to?(:post_process)
+                  callback.post_process(store_response, result)
+                end
+                
+                if block_given?
+                  yield :store, store_response
+                end                
+              end
+              
+            end
+          #end each binding
           end
       end
       

data/lib/pho/file_management.rb

@@ -3,7 +3,6 @@ module Pho
   module FileManagement
 
     #TODO: move files into hidden directory
-    #TODO: add directory traversal
     class AbstractFileManager
     
       attr_reader :dir
@@ -21,12 +20,12 @@ module Pho
       end
   
       #Store all files that match the file name in directory
-      def store()
-        files_to_store = new_files()
+      def store(recursive=false)
+        files_to_store = new_files(recursive)
         files_to_store.each do |filename|
           file = File.new(filename)
           store_file(file, filename)
-        end    
+        end
       end
   
       #Retry anything known to have failed
@@ -41,8 +40,7 @@ module Pho
       end
             
       #Reset the directory to clear out any previous statuses
-      #Store can  also be reset at the same time: use with care!
-      def reset(reset_store=false)
+      def reset()
         Dir.glob( File.join(@dir, "*.#{@fail_suffix}") ).each do |file|
           File.delete(file)
         end
@@ -50,11 +48,26 @@ module Pho
           File.delete(file)
         end         
       end
-                    
+           
+      #List any new files in the directory
+      def new_files(recursive=false)
+        newfiles = Array.new
+        list(recursive).each do |file|
+          
+          ok_file = get_ok_file_for(file)
+          fail_file = get_fail_file_for(file)
+          if !( File.exists?(ok_file) or File.exists?(fail_file) )
+            newfiles << file          
+          end
+            
+        end
+        return newfiles
+      end
+                     
       #List failures
-      def failures()
+      def failures(recursive=false)
         fails = Array.new
-        Dir.glob( File.join(@dir, "*") ).each do |file|
+        list(recursive).each do |file|
           if File.extname(file) != ".#{@fail_suffix}" && File.extname(file) != ".#{@ok_suffix}"
             fails << file if File.exists?( get_fail_file_for(file) )  
           end          
@@ -63,9 +76,9 @@ module Pho
       end
       
       #List successes
-      def successes()
+      def successes(recursive=false)
         successes = Array.new
-        Dir.glob( File.join(@dir, "*") ).each do |file|
+        list(recursive).each do |file|
           if File.extname(file) != ".#{@fail_suffix}" && File.extname(file) != ".#{@ok_suffix}"
              successes << file if File.exists?( get_ok_file_for(file) )
           end
@@ -73,25 +86,25 @@ module Pho
         return successes
       end
             
-      #Summarize the state of the collection to the provied IO object
+      #Summarize the state of the collection to the provided IO object
       #Creates a simple report
-      def summary()
-         failures = failures()
-         successes = successes()
-         newfiles = new_files()
+      def summary(recursive=false)
+         failures = failures(recursive)
+         successes = successes(recursive)
+         newfiles = new_files(recursive)
          total = failures.size + successes.size + newfiles.size
-         summary = "#{@dir} contains #{total} files: #{successes.size} stored, #{failures.size} failed, #{newfiles.size} new"
+         summary = "#{@dir} #{recursive ? " and sub-directories" : ""} contains #{total} files: #{successes.size} stored, #{failures.size} failed, #{newfiles.size} new"
          return summary     
       end
       
       def get_fail_file_for(filename)
         ext = File.extname(filename)
-        return filename.gsub(/#{ext}/, ".#{@fail_suffix}")              
+        return filename.gsub(/#{ext}$/, ".#{@fail_suffix}")              
       end
       
       def get_ok_file_for(filename)
         ext = File.extname(filename)        
-        return filename.gsub(/#{ext}/, ".#{@ok_suffix}")
+        return filename.gsub(/#{ext}$/, ".#{@ok_suffix}")
       end
                  
     end