pho 0.0.1

data/README

@@ -0,0 +1,56 @@
+The Pho ruby module provides a lightweight Ruby client library for interacting with the Talis 
+Platform API (http://n2.talis.com/wiki/Platform_API).
+
+== Author
+ 
+Leigh Dodds (leigh.dodds@talis.com)
+
+== Download
+
+The latest version of this library can be downloaded from:
+
+http://pho.rubyforge.net
+
+== Usage
+
+To use Pho you can:
+
+   require 'pho'
+
+And then create an instance of the appropriate class, e.g Store:
+
+   store = Pho::Store.new("http://api.talis.com/testing", "user", "pass")
+   response = store.describe("http://www.example.org")
+
+For more detailed examples consult the documentation for the Store class
+
+== Control over HTTP interactions
+
+Pho is dependent on the HTTPClient module and all HTTP interactions are delegated to 
+an instance of the HTTPClient class. In circumstances where greater control over the 
+HTTP interaction is required, e.g. to configure proxy servers, etc, an existing instance of 
+this class can be provided, e.g:
+
+  client = HTTPClient.new
+  => configure client as required
+  store = Pho::Store.new("http://api.talis.com/testing", "user", "pass", client)
+  => pass instance of client as parameter
+  response = store.describe("http://www.example.org")
+
+== License
+
+Copyright 2009 Leigh Dodds 
+ 
+Licensed under the Apache License, Version 2.0 (the "License"); 
+you may not use this file except in compliance with the License. 
+  
+You may obtain a copy of the License at 
+  
+http://www.apache.org/licenses/LICENSE-2.0 
+  
+Unless required by applicable law or agreed to in writing, 
+software distributed under the License is distributed on an "AS IS" BASIS, 
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
+  
+See the License for the specific language governing permissions and limitations 
+under the License. 

data/Rakefile

@@ -0,0 +1,63 @@
+require 'rake'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/testtask'
+require 'rake/clean'
+
+NAME = "pho"
+VER = "0.0.1"
+
+RDOC_OPTS = ['--quiet', '--title', 'Pho (Talis Platform Client) Reference', '--main', 'README']
+
+PKG_FILES = %w( README Rakefile ) + 
+  Dir.glob("{bin,doc,tests,lib}/**/*")
+
+CLEAN.include ['*.gem', 'pkg']  
+SPEC =
+  Gem::Specification.new do |s|
+    s.name = NAME
+    s.version = VER
+    s.platform = Gem::Platform::RUBY
+    s.has_rdoc = true
+    s.extra_rdoc_files = ["README"]
+    s.rdoc_options = RDOC_OPTS
+    s.summary = "Ruby client for the Talis Platform"
+    s.description = s.summary
+    s.author = "Leigh Dodds"
+    s.email = 'leigh.dodds@talis.com'
+    s.homepage = 'http://pho.rubyforge.net'
+    s.rubyforge_project = 'pho'
+    s.files = PKG_FILES
+    s.require_path = "lib" 
+    s.bindir = "bin"
+    s.test_file = "tests/ts_pho.rb"
+    s.add_dependency("httpclient", ">= 2.1.3.1")
+    s.add_dependency("json", ">= 1.1.3")
+    s.add_dependency("mocha", ">= 0.9.5")
+  end
+      
+Rake::GemPackageTask.new(SPEC) do |pkg|
+    pkg.need_tar = true
+end
+
+Rake::RDocTask.new do |rdoc|
+    rdoc.rdoc_dir = 'doc/rdoc'
+    rdoc.options += RDOC_OPTS
+    rdoc.main = "README"
+    rdoc.rdoc_files.add ["README"]
+end
+
+
+Rake::TestTask.new do |test|
+  test.test_files = FileList['tests/tc_*.rb']
+end
+
+task :install do
+  sh %{rake package}
+  sh %{sudo gem install pkg/#{NAME}-#{VER}}
+end
+
+
+task :uninstall => [:clean] do
+  sh %{sudo gem uninstall #{NAME}}
+end

data/lib/pho.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+require 'httpclient'
+require 'json'
+
+require 'pho/etags'
+require 'pho/store'
+require 'pho/rdf_collection'
+
+module Pho
+
+  ACCEPT_RDF = {"Accept" => "application/rdf+xml"}.freeze   
+  ACCEPT_JSON = { "Accept" => "application/json" }.freeze
+  
+  RDF_XML = {"Content-Type"=>"application/rdf+xml"}.freeze
+
+  JOB_RESET = "http://schemas.talis.com/2006/bigfoot/configuration#ResetDataJob".freeze
+  JOB_SNAPSHOT = "http://schemas.talis.com/2006/bigfoot/configuration#SnapshotJob".freeze
+  JOB_REINDEX = "http://schemas.talis.com/2006/bigfoot/configuration#ReindexJob".freeze   
+     
+
+end

data/lib/pho/etags.rb

@@ -0,0 +1,54 @@
+require 'yaml'
+
+module Pho
+  
+  #Simple mechanism for managing etags
+  class Etags
+  
+    attr_reader :file, :saved
+    
+    def initialize(file = nil)
+      @file = file
+      @saved = true
+      @tags = Hash.new
+      if @file != nil
+          @tags = YAML::load(@file)[0]
+      end
+    end
+    
+    def save(other=nil)
+
+      if (other != nil)
+        other.write( @tags.to_yaml() )
+        return
+      else
+        if (!saved && @file != nil )
+            @file.write( @tags.to_yaml() )
+            @file.close           
+        end        
+      end
+                        
+    end
+    
+    def add(uri, tag)     
+      if (uri != nil && tag != nil)
+        @tags[uri] = tag
+        @saved = false        
+      end
+    end  
+    
+    def add_from_response(uri, response)
+      add(uri, response.header["ETag"][0])
+    end
+    
+    def get(uri)
+      return @tags[uri]
+    end
+    
+    def has_tag?(uri)
+      return @tags.has_key?(uri)
+    end
+    
+  end  
+  
+end

data/lib/pho/rdf_collection.rb

@@ -0,0 +1,107 @@
+module Pho
+  
+  attr_reader :dir
+  attr_reader :store
+  
+  # Provides a simple mechanism for managing a directory of RDF/XML documents
+  # and uploading them to platform store.
+  #
+  # Allows a collection to be mirrored into the platform
+  class RDFCollection
+    
+    RDF = "rdf".freeze
+    OK = "ok".freeze
+    FAIL = "fail".freeze
+    
+    def initialize(store, dir, rdf_suffix=RDF, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
+      @store = store
+      @dir = dir
+      @sleep = sleep
+      @rdf_suffix = rdf_suffix
+      @ok_suffix = ok_suffix
+      @fail_suffix = fail_suffix
+    end
+
+    #Store all files that match the file name in directory
+    def store()
+      files_to_store = new_files()
+      files_to_store.each do |filename|
+        file = File.new(filename)
+        response = @store.store_file(file)
+      end    
+    end
+
+    #Retry anything known to have failed
+    def retry_failures()
+      #TODO
+    end
+        
+    #Reset the directory to clear out any previous statuses
+    #Store can  also be reset at the same time: use with care!
+    def reset(reset_store=false)
+      Dir.glob( File.join(@dir, "*.#{@fail_suffix}") ).each do |file|
+        File.delete(file)
+      end
+      Dir.glob( File.join(@dir, "*.#{@ok_suffix}") ).each do |file|
+        File.delete(file)
+      end         
+    end
+            
+    #List files being managed
+    def list()
+        return Dir.glob( File.join(@dir, "*.#{@rdf_suffix}") )  
+    end
+    
+    #List failures
+    def failures()
+      fails = Array.new
+      Dir.glob( File.join(@dir, "*.#{@fail_suffix}") ).each do |file|
+        fails << file.gsub(/\.#{@fail_suffix}/, ".#{@rdf_suffix}")
+      end
+      return fails
+    end
+    
+    #List successes
+    def successes()
+      successes = Array.new
+      Dir.glob( File.join(@dir, "*.#{@ok_suffix}") ).each do |file|
+        successes << file.gsub(/\.#{@ok_suffix}/, ".#{@rdf_suffix}")
+      end
+      return successes
+    end
+    
+    #List any new files in the directory
+    def new_files()
+      newfiles = Array.new
+      Dir.glob( File.join(@dir, "*.#{@rdf_suffix}") ) do |file|
+        ok_file = get_ok_file_for(file)
+        fail_file = get_fail_file_for(file)
+        if !( File.exists?(ok_file) or File.exists?(fail_file) )
+          newfiles << file          
+        end
+      end
+      return newfiles
+    end
+    
+    #Summarize the state of the collection to the provied IO object
+    #Creates a simple report
+    def summary()
+       failures = failures()
+       successes = successes()
+       newfiles = new_files()
+       total = failures.size + successes.size + newfiles.size
+       summary = "#{@dir} contains #{total} files: #{successes.size} stored, #{failures.size} failed, #{newfiles.size} new"
+       return summary     
+    end
+    
+    def get_fail_file_for(filename)
+      return filename.gsub(/\.#{@rdf_suffix}/, ".#{@fail_suffix}")              
+    end
+    
+    def get_ok_file_for(filename)
+      return filename.gsub(/\.#{@rdf_suffix}/, ".#{@ok_suffix}")
+    end
+          
+  end
+    
+end  

data/lib/pho/store.rb

@@ -0,0 +1,389 @@
+module Pho
+
+#TODO:
+#
+# Changesets
+# Multisparql
+#  
+# Conditional deletions
+# If-Modified-Since support
+# Robustness in uri fetching
+#
+# RDOC
+  
+  # The Store class acts as a lightweight client interface to the Talis Platform API  
+  # (http://n2.talis.com/wiki/Platform_API). The class provides methods for interacting 
+  # with each of the core platform services, e.g. retrieving and storing RDF, performing
+  # searches, SPARQL queries, etc. 
+  # 
+  # == Usage
+  # 
+  #   store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")
+  #   store.store_file( File.new("/tmp/example.rdf") )
+  #   store.store_url( "http://www.example.org/example.rdf" )
+  #   store.describe( "http://www.example.org/thing" )
+  #   store.reset
+  #
+  # == Examples
+  # 
+  class Store
+  
+    #Retrieve the HTTPClient instance being used by this object
+    attr_reader :client
+    #Retrieve the admin username configured in this instance
+    attr_reader :username
+    #Retrieve the base uri of this store
+    attr_reader :storeuri
+  
+    # Create an instance of the store class
+    #
+    # storeuri::  base uri for the Platform store to be accessed
+    # username::  admin username, may be nil
+    # password::  admin password, may be nil
+    # client::  an instance of HTTPClient
+    def initialize(storeuri, username=nil, password=nil, client = HTTPClient.new() )
+      @storeuri = storeuri.chomp("/")
+      @username = username
+      @password = password
+      @client = client
+      set_credentials(username, password) if username or password
+    end
+    
+    # Set credentials that this store will use when carrying out authorization
+    #
+    # username:: admin username
+    # password:: admin password    
+    def set_credentials(username, password)
+      @client.set_auth(@storeuri, username, password)
+    end
+    
+    # Build a request uri, by concatenating it with the base uri of the store
+    # uri:: relative URI to store service, e.g. "/service/sparql"
+    def build_uri(uri)
+      if (uri.start_with?(@storeuri))
+        return uri
+      end
+      if uri.start_with?("/")
+        return @storeuri + uri
+      else
+        return @storeuri + "/" + uri
+      end
+    end
+    
+    #############
+    # METABOX
+    #############
+    
+    # Store some RDF in the Metabox associated with this store
+    # data:: a String containing the data to store
+    def store_data(data)
+      u = build_uri("/meta")
+      response = @client.post(u, data, RDF_XML )
+      return response
+    end    
+    
+    # Store the contents of a File (or any IO stream) in the Metabox associated with this store
+    # The client does not support streaming submissions of data, so the stream will be fully read before data is submitted to the platform
+    # file:: an IO object      
+    def store_file(file)
+      data = file.read()
+      file.close()
+      return store_data(data)
+    end
+    
+    # Retrieve RDF data from the specified URL and store it in the Store Metabox
+    #
+    # An Accept header of "application/rdf+xml" will be sent in the request to support retrieval of RDF from
+    # URLs that support Content Negotiation.
+    #
+    # u:: the url of the data
+    # parameters:: a Hash of url parameters to pass when requesting data from the specified URL
+    def store_url(u, parameters=nil)
+      
+      headers = ACCEPT_RDF.clone()
+      dataresp = @client.get(u, parameters, headers )
+      
+      #TODO make this more robust
+      if dataresp.status != 200
+          throw  
+      end
+      
+      return store_data(dataresp.content)
+      
+    end
+  
+    # Retrieve an RDF description of a specific URI. The default behaviour will be to retrieve an RDF/XML document, but other formats can be 
+    # requested, as supported by the Talis Platform. E.g. application/json 
+    #
+    # uri:: the URI of the resource to describe
+    # format:: the preferred response format
+    # etags:: an instance of the Pho::Etags class to support conditional GETs
+    # if_match:: specify true to retrieve data only if the version matches a known ETag, false to perform a Conditional GET
+    #
+    # Note that this method is different from sparql_describe in that it is intended to be used to generate a description of 
+    # a single URI, using an separated service exposed by the Platform. This service is optimised for retrieval of descriptions for 
+    # single resources and supports HTTP caching and conditional retrieval. The sparql_describe method should be used to submit
+    # more complex DESCRIBE queries to the Platform, e.g. to generate descriptions of resources matching a particular graph pattern. 
+    def describe(uri, format="application/rdf+xml", etags=nil, if_match=false)
+      u = self.build_uri("/meta")
+      headers = {"Accept" => format}
+      headers = configure_headers_for_conditional_get("#{u}?about=#{uri}", headers, etags, if_match)
+      response = @client.get(u, {"about" => uri}, headers )
+      record_etags("#{u}?about=#{uri}", etags, response)        
+      return response
+    end
+  
+    #############
+    # SERVICES
+    #############
+    
+    #Perform a SPARQL DESCRIBE query.
+    #
+    # query:: the SPARQL query
+    # format:: the preferred response format
+    def sparql_describe(query, format="application/rdf+xml")
+      return sparql(query, format)
+    end
+
+    #Perform a SPARQL CONSTRUCT query.
+    #
+    # query:: the SPARQL query
+    # format:: the preferred response format        
+    def sparql_construct(query, format="application/rdf+xml")
+      return sparql(query, format)
+    end
+    
+    #Perform a SPARQL ASK query.
+    #
+    # query:: the SPARQL query
+    # format:: the preferred response format    
+    def sparql_ask(query, format="application/sparql-results+xml")
+      return sparql(query, format)
+    end
+    
+    #Perform a SPARQL SELECT query.
+    #
+    # query:: the SPARQL query
+    # format:: the preferred response format    
+    def sparql_select(query, format="application/sparql-results+xml")
+      return sparql(query, format)
+    end
+    
+    #Perform a SPARQL query
+    #
+    # query:: the SPARQL query
+    # format:: the preferred response format    
+    def sparql(query, format)
+      u = self.build_uri("/services/sparql")
+      params = {}
+      params["query"] = query
+      headers = {}
+      headers["Accept"] = format
+      response = @client.get(u, params, headers)        
+    end    
+    
+    # Search the Metabox indexes.
+    #
+    # query:: the query to perform. See XXXX for query syntax
+    # params:: additional query parameters (see below)
+    #
+    # The _params_ hash can contain the following values:
+    # * *max*: The maximum number of results to return (default is 10)
+    # * *offset*: Offset into the query results (for paging; default is 0)
+    # * *sort*: ordered list of fields to be used when applying sorting    
+    # * *xsl-uri*: URL of an XSLT transform to be applied to the results, transforming the default RSS 1.0 results format into an alternative representation    
+    # * *content-type*: when applying an XSLT transform, the content type to use when returning the results
+    #
+    # Any additional entries in the _params_ hash will be passed through to the Platform. 
+    # These parameters will only be used when an XSLT transformation is being applied, in which case they 
+    # will be provided as parameters to the stylesheet. 
+    def search(query, params=nil)
+      u = self.build_uri("/items")
+      search_params = get_search_params(u, query, params)
+      response = @client.get(u, search_params, nil)
+      return response
+      
+    end
+    
+    # Perform a facetted search against the Metabox indexes.
+    #
+    # query:: the query to perform. See XXXX for query syntax
+    # facets:: an ordered list of facets to be used
+    # params:: additional query parameters (see below)
+    #
+    # The _params_ hash can contain the following values:
+    # * *top*: the maximum number of results to return for each facet
+    # * *output*: the preferred response format, can be html or xml (the default)
+    def facet(query, facets, params=nil)
+      if facets == nil or facets.empty?
+        #todo
+        throw
+      end
+      u = self.build_uri("/services/facet")      
+      search_params = get_search_params(u, query, params)
+      search_params["fields"] = facets.join(",")
+      response = @client.get(u, search_params, nil)
+      return response                             
+    end
+        
+    def get_search_params(u, query, params)
+      if params != nil
+        search_params = params.clone()
+      else
+        search_params = Hash.new  
+      end
+      search_params["query"] = query
+      return search_params      
+    end
+    
+    # Augment an RSS feed that can be retrieved from the specified URL, against data in this store
+    #
+    # uri:: the URL for the RSS 1.0 feed
+    def augment_uri(uri)
+      u = self.build_uri("/services/augment")
+      response = @client.get(u, {"data-uri" => uri}, nil)
+      return response          
+    end
+    
+    # Augment an RSS feed against data int this store by POSTing it to the Platform
+    #
+    # data:: a String containing the RSS feed
+    def augment(data)
+      u = self.build_uri("/services/augment")
+      response = @client.post(u, data, nil)
+      return response
+    end
+        
+    #Added appropriate http header for conditional get requests
+    def configure_headers_for_conditional_get(u, headers, etags, if_match)
+      if etags != nil && etags.has_tag?(u)
+         if if_match
+           headers["If-Match"] = etags.get(u)  
+         else
+           headers["If-None-Match"] = etags.get(u)
+         end                  
+      end
+      return headers      
+    end
+            
+    def record_etags(u, etags, response)
+      if (etags != nil && response.status = 200)        
+        etags.add_from_response(u, response)  
+      end      
+    end
+   
+    
+    #############
+    # CONTENTBOX
+    #############
+        
+    # Store an item in the Contentbox for this store
+    #
+    # f:: a File or other IO object from which data will be read
+    # mimetype:: the mimetype of the object to record in the Platform
+    # uri:: the URI at which to store the item (relative to base uri for the store). If nil, then a URI will be assigned by the Platform
+    #
+    # When a _uri_ is not specified, then the Platform will return a 201 Created response with a Location header containing the URI of the 
+    # newly stored item. If a URI is specified then a successful request will result in a 200 OK response.
+    def upload_item(f, mimetype, uri=nil)
+      data = f.read()
+      f.close()
+      headers = {"Content-Type" => mimetype}
+      if uri == nil
+        u = self.build_uri("/items")        
+        response = @client.post(u, data, headers)
+      else
+        if !uri.start_with?(@storeuri)
+          uri = build_uri(uri)
+        end
+        response = @client.put(uri, data, headers)
+      end      
+      return response
+    end
+   
+    # Delete an item from the Contentbox in this Store
+    # uri:: the URL of the item, can be relative
+    # TODO: conditional deletes
+    def delete_item(uri)
+      if !uri.start_with?(@storeuri)
+        uri = build_uri(uri)
+      end
+      return @client.delete(uri)
+    end    
+    
+    # Get an item from the Contebtbox. 
+    # uri:: the URL of the item, can be relative.
+    # 
+    # If the provided URL of the item is not in the Contentbox, then the response will be a redirect to the 
+    # RDF description of this item, as available from the Metabox.
+    #
+    # TODO: document etags, redirects
+    def get_item(uri, etags=nil, if_match=false)
+      u = self.build_uri(uri)
+      headers = Hash.new
+      headers = configure_headers_for_conditional_get("#{u}", headers, etags, if_match)      
+      response = @client.get(u, nil, headers)
+      record_etags("#{u}", etags, response)
+      return response
+    end
+    
+    #############
+    # JOBS
+    #############
+  
+    # Construct an RDF/XML document containing a job request for submitting to the Platform.
+    #
+    # t:: a Time object, specifying the time at which the request should be carried out
+    # joburi:: the URI for the JobType that should be created
+    # label:: a label for this job.     
+    def build_job_request(t, joburi, label)
+      time = t.strftime("%Y-%m-%dT%H:%M:%SZ")
+      data = "<rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" "
+      data << " xmlns:rdfs=\"http://www.w3.org/2000/01/rdf-schema#\" " 
+      data << " xmlns:bf=\"http://schemas.talis.com/2006/bigfoot/configuration#\"> " 
+      data << " <bf:JobRequest>"
+      data << "   <rdfs:label>#{label}</rdfs:label>"    
+      data << "   <bf:jobType rdf:resource=\"#{joburi}\"/>"
+      data << "   <bf:startTime>#{time}</bf:startTime>"
+      data << " </bf:JobRequest>"
+      data << "</rdf:RDF>"
+      return data      
+    end    
+    
+    def reset(t=Time.now)
+      return submit_job(JOB_RESET, "Reset my store", t)
+    end
+    
+    def reindex(t=Time.now)
+      return submit_job(JOB_REINDEX, "Reindex my store", t)      
+    end
+    
+    def snapshot(t=Time.now)
+      return submit_job(JOB_SNAPSHOT, "Snapshot my store", t)
+    end  
+  
+    def submit_job(joburi, label, t=Time.now)
+        u = build_uri("/jobs")
+        data = build_job_request(t, joburi, label)
+        response = @client.post(u, data, RDF_XML )
+        return response          
+    end
+    
+    #############
+    # ADMIN
+    #############
+    
+    def status()
+      u = build_uri("/config/access-status")
+      response = @client.get_content(u, nil, ACCEPT_JSON )
+      json = JSON.parse( response )
+      state = Hash.new
+      state["retryInterval"] = json[u.to_s]["http:\/\/schemas.talis.com\/2006\/bigfoot\/configuration#retryInterval"][0]["value"]
+      state["statusMessage"] = json[u.to_s]["http:\/\/schemas.talis.com\/2006\/bigfoot\/configuration#statusMessage"][0]["value"]
+      state["accessMode"] = json[u.to_s]["http:\/\/schemas.talis.com\/2006\/bigfoot\/configuration#accessMode"][0]["value"]
+      return state
+    end
+                 
+  end
+
+end