pho 0.7.8 → 0.7.9

data/lib/pho.rb

@@ -4,7 +4,9 @@ require 'json'
 require 'yaml'
 require 'date'
 require 'rexml/document'
-require 'md5'
+#require 'md5'
+require "digest/md5"
+require "ftools"
 
 #RDF.rb
 require 'rdf'
@@ -31,6 +33,7 @@ require 'pho/enrichment'
 require 'pho/command_line'
 require 'pho/oai'
 require 'pho/converter'
+require 'pho/upload.rb'
 
 if RUBY_VERSION < "1.8.7"
   class String

data/lib/pho/file_management.rb

@@ -1,7 +1,31 @@
 module Pho
   
+  #This module provides support for managing local directories of files and mirroring them
+  #into either the metabox or contentbox of a Talis Platform store
   module FileManagement
 
+    #Base class for the management of collections of files
+    #
+    #A collection is considered to be a directory (and sub-directory) of files. Depending on 
+    #the type of collection manager used (e.g. +Pho::FileManagement::FileManager+ or 
+    #+Pho::FileManagement::RDFManager+) these may be collections of any kind of file, or just 
+    #RDF serialisations.
+    #
+    #The basic mechanism has support for:
+    # * tracking whether a file has been successfully loaded into a store (or not)
+    # * identifying all new files, just those that have been successfully stored, or failed
+    # * identifying files that have been changed locally since last stored
+    # * storage of any of these classes of files either by uploading into the contentbox or 
+    #   storing in the metabox
+    # * resumable uploads
+    # * retrying uploads
+    # * traversal across a directory structure to manage a complete set of files
+    #
+    #Files are tracked locally by keeping some hidden files in the same directory system as the 
+    #files being tracked. These are currently all managed in a hidden directory (".pho") that is 
+    #automatically created. To access error messages from uploads look for the corresponding "fail file" 
+    #for the relevant file. E.g. /foo/file.txt will be tracked in /foo/.pho, and will have either an 
+    #OK file (/foo/.pho/file.txt.ok) or a fail file (/foo/.pho/file.txt.fail). 
     class AbstractFileManager
     
       attr_reader :dir
@@ -162,7 +186,12 @@ module Pho
         relative_path = relative_path.gsub(base, "#{TRACKING_DIR}/#{base}")
         return "#{@dir}#{relative_path}.#{@ok_suffix}"              
       end
-                 
+       
+      def create_tracking_dir(filename)
+        path = filename.split("/")[0..-2].join("/")  
+        Dir.mkdir("#{path}/#{TRACKING_DIR}") unless File.exists?("#{path}/#{TRACKING_DIR}")
+      end
+                
     end
     
   end

data/lib/pho/file_manager.rb

@@ -4,6 +4,13 @@ module Pho
     
   module FileManagement
 
+    #Manages a collection of files in a directory structure, uploading them to 
+    #the contentbox of a platform store
+    #
+    #Allows the base directory of the upload to be specified, giving some flexibility
+    #on how the files are published via the contentbox. By default the files will
+    #be rooted in /items/ but by specifying the base parameter in the construct, this 
+    #can be altered to, e.g. /items/foo
     class FileManager < AbstractFileManager
    
       attr_reader :base

data/lib/pho/rdf_collection.rb

@@ -2,10 +2,10 @@ module Pho
   
   module FileManagement
 
-      # Provides a simple mechanism for managing a directory of RDF/XML documents
-      # and uploading them to platform store.
+      #Provides a simple mechanism for managing a directory of RDF documents
+      #and uploading them to platform store.
       #
-      # Allows a collection to be mirrored into the platform
+      #Currently supports RDF, Turtle and NTriples
       class RDFManager < AbstractFileManager
       
         RDF = "rdf".freeze
@@ -40,7 +40,7 @@ module Pho
           else
             response = @store.store_file(file)
           end
-                    
+          create_tracking_dir(filename)                              
           if (response.status < 300 )
             File.open(get_ok_file_for(filename), "w") do |file|
               file.print( "OK" )            

data/lib/pho/upload.rb

@@ -0,0 +1,156 @@
+module Pho
+  
+  module FileManagement
+    
+    #Default statement handler, does nothing
+    class StatementHandler
+      def handle(statement)
+        return statement
+      end
+    end
+    
+    #Remove bnodes from the input data by assigning URIs to them
+    #
+    #This implementation generates a simple hexdigest based on the node id
+    #and uses that to construct a uri based on a base uri provided in the 
+    #constructor
+    class BNodeRewritingHandler
+      # base:: base uri for URIs generated for blank nodes
+      def initialize(base)
+        @base = base
+      end
+      
+      def handle(statement)
+        if !statement.has_blank_nodes?
+          return statement
+        end
+        subject = statement.subject
+        if subject.anonymous?
+          subject = RDF::URI.new( assign_uri(subject) )
+        end
+        object = statement.object
+        if object.anonymous?
+          object = RDF::URI.new( assign_uri(object) )
+        end
+        return RDF::Statement.new(subject, statement.predicate, object)
+      end
+      
+      #FIXME semantics for this is wrong if nodeIds are reused across
+      #datasets
+      def assign_uri(node)
+        return "#{@base}/#{Digest::MD5.hexdigest( node.id )}#self"
+      end
+      
+    end
+    
+    #Supports splitting RDF data files into smaller chunks of ntriples     
+    class FileSplitter
+      
+      attr_reader :dir, :triples, :handler
+      
+      DEFAULT_CHUNK_SIZE = 10000
+      
+      #Create a file splitter instance
+      #
+      #  dir:: temporary directory into which split files should be written
+      #  triples:: number of triples per split file
+      #  handler:: statement handler to allow pre-processing of statements
+      def initialize(dir="/tmp", triples=DEFAULT_CHUNK_SIZE, 
+          handler=Pho::FileManagement::StatementHandler.new)
+        @dir = dir
+        @triples = triples
+        @handler = handler
+      end
+            
+      #Split a single file, in any parseable RDF format into smaller
+      #chunks of ntriples. Chunked files are stored in default temporary
+      #directory for this instance
+      #
+      #  filename:: name of the file to split
+      #  format:: input format, default is :ntriples
+      def split_file(filename, format=:ntriples)
+        
+        basename = File.basename(filename, ".#{filename.split(".").last}")
+        count = 0
+        stmts = []
+        RDF::Reader.for(format).new(File.new(filename)) do |reader|
+          reader.each_statement do |statement|            
+            count += 1
+            stmts << @handler.handle( statement ) 
+            if count % @triples == 0
+              RDF::Writer.open( File.join(@dir, "#{basename}_#{count}.nt") ) do |writer|
+                stmts.each do |s|
+                  writer << s
+                end
+              end              
+              stmts = []              
+            end
+          end
+        end
+        if !stmts.empty?
+          RDF::Writer.open( File.join(@dir, "#{basename}_#{count}.nt") ) do |writer|
+            stmts.each do |s|
+              writer << s
+            end
+          end
+        end
+      end
+
+      #Split a list of files into smaller chunks
+      #
+      #  list_of_filenames:: array of filenames
+      #  format:: format of the files, default is :ntriples      
+      def split_files(list_of_filenames, format=:ntriples)
+        list_of_filenames.each do |name|
+          split_file(name, format)
+        end
+      end  
+                
+    end
+        
+    class Util
+
+      #Take a directory of files, copy them to temporary directory, splitting
+      #where necessary, in preparation for uploading to a platform store.
+      #
+      #Source directory is scanned for ntriple, turtle and RDF/XML files. All of 
+      #these are automatically chunked into 10,000 triple chunks and re-serialized 
+      #as ntriples
+      #
+      #BNodes are automatically re-written to full uris.
+      #
+      #  store:: Pho::Store into which data will be posted. Used to normalizing bnodes
+      #  src_dir:: directory containing source data.      
+      def Util.prepare_platform_upload(store, src_dir, collection_dir, 
+          triples=FileSplitter::DEFAULT_CHUNK_SIZE)
+          
+          handler = BNodeRewritingHandler.new( store.build_uri("/items") )
+          splitter = FileSplitter.new(collection_dir, triples, handler )
+          
+          formats = [ ["*.rdf", :rdfxml], ["*.nt", :ntriples], ["*.ttl", :turtle] ]
+          formats.each do |format|
+            
+            files = Dir.glob( File.join(src_dir, format[0] ) )
+            splitter.split_files(files, format[1] )
+            
+          end
+          return true
+          
+      end
+      
+      #Prepares a batch of files for uploading into the platform, then posts
+      #that collection to the designated store
+      #
+      #Returns an RDFManager instance that can be inspected to check for successes
+      def Util.prepare_and_store_upload(store, src_dir, collection_dir, 
+          triples=FileSplitter::DEFAULT_CHUNK_SIZE)
+          
+        prepare_platform_upload(store, src_dir, collection_dir)
+        collection = Pho::FileManagement::RDFManager.new(store, collection_dir)
+        collection.store()
+        return collection
+      end
+      
+    end
+  end
+end

data/tests/tc_bnodehandler.rb

@@ -0,0 +1,42 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'pho'
+require 'test/unit'
+require 'mocha'
+
+class BNodeRewritingHandlerTest < Test::Unit::TestCase
+  
+  def test_uris_are_not_changed()
+    handler = Pho::FileManagement::BNodeRewritingHandler.new("http://example.org")
+    
+    subject = RDF::URI.new("http://example.org/subject")
+    object = RDF::URI.new("http://example.org/object")
+    
+    statement = RDF::Statement.new(subject, RDF::RDFS.label, object)
+    
+    handled = handler.handle(statement)
+    
+    assert_equal( statement.subject, handled.subject)
+    assert_equal( statement.predicate, handled.predicate)
+    assert_equal( statement.object, handled.object)
+    
+  end
+  
+  def test_uri_assignment()
+    handler = Pho::FileManagement::BNodeRewritingHandler.new("http://example.org")
+    
+    subject = RDF::Node.new
+    object = RDF::Node.new
+    
+    statement = RDF::Statement.new(subject, RDF::RDFS.label, object)    
+    handled = handler.handle(statement)
+    
+    assert_equal(statement.predicate, handled.predicate)
+    assert_equal(false, handled.subject.anonymous?)
+    assert_equal(false, handled.object.anonymous?)
+    assert_equal("http://example.org/#{Digest::MD5.hexdigest( subject.id )}#self", 
+      handled.subject.to_s)
+    assert_equal("http://example.org/#{Digest::MD5.hexdigest( object.id )}#self", 
+      handled.object.to_s)    
+    
+  end
+end

data/tests/tc_filesplitter.rb

@@ -0,0 +1,51 @@
+$:.unshift File.join(File.dirname(__FILE__), "..", "lib")
+require 'pho'
+require 'test/unit'
+require 'mocha'
+
+class FileSplitterTest < Test::Unit::TestCase
+  
+  def setup()
+    Dir.mkdir("/tmp/pho") unless File.exists?("/tmp/pho")
+    Dir.mkdir("/tmp/pho/split") unless File.exists?("/tmp/pho/split")
+    Dir.mkdir("/tmp/pho/split/tmp") unless File.exists?("/tmp/pho/split/tmp")
+    
+    RDF::Writer.open("/tmp/pho/split/large.nt") do |writer|
+      31.times do 
+        writer << RDF::Statement.new( 
+          RDF::Resource.new("http://www.example.org"), 
+          RDF::RDFS.label,
+          RDF::Literal.new("This is a test")
+          )
+      end      
+    end
+    
+  end
+  
+  def teardown()
+    Dir.glob("/tmp/pho/split/tmp/*.*") do |file|
+      File.delete(file)
+    end    
+    Dir.glob("/tmp/pho/split/*.*") do |file|
+      File.delete(file)
+    end    
+    delete("/tmp/pho/split/tmp")
+    delete("/tmp/pho/split")
+  end
+  
+  def delete(dir)
+    Dir.delete(dir) if File.exists?(dir)
+  end
+  
+  def test_split_file()
+    splitter = Pho::FileManagement::FileSplitter.new("/tmp/pho/split/tmp", 10)
+    splitter.split_file("/tmp/pho/split/large.nt")
+    assert_equal(4, Dir.glob("/tmp/pho/split/tmp/large*.nt").size )
+    assert_equal(true, File.exists?("/tmp/pho/split/tmp/large_10.nt"))
+    assert_equal(true, File.exists?("/tmp/pho/split/tmp/large_20.nt"))
+    assert_equal(true, File.exists?("/tmp/pho/split/tmp/large_30.nt"))
+    assert_equal(true, File.exists?("/tmp/pho/split/tmp/large_31.nt"))
+          
+  end
+  
+end

data/tests/ts_pho.rb

@@ -23,4 +23,6 @@ require 'tc_changesets.rb'
 require 'tc_converter.rb'
 require 'tc_enrichment.rb'
 require 'tc_command_line.rb'
-require 'tc_oai.rb'
+require 'tc_oai.rb'
+require 'tc_filesplitter.rb'
+require 'tc_upload.rb'

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: pho
 version: !ruby/object:Gem::Version 
-  hash: 19
-  prerelease: false
+  hash: 17
+  prerelease: 
   segments: 
   - 0
   - 7
-  - 8
-  version: 0.7.8
+  - 9
+  version: 0.7.9
 platform: ruby
 authors: 
 - Leigh Dodds
@@ -15,8 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-20 00:00:00 +01:00
-default_executable: 
+date: 2011-09-21 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: httpclient
@@ -82,6 +81,48 @@ dependencies:
         version: "1.16"
   type: :runtime
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rdf
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rdf-json
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: rdf-raptor
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id007
 description: Ruby client for the Talis Platform
 email: leigh.dodds@talis.com
 executables: 
@@ -104,6 +145,7 @@ files:
 - doc/rdoc/files/lib/pho/store_rb.html
 - doc/rdoc/files/lib/pho/status_rb.html
 - doc/rdoc/files/lib/pho/resource_hash_rb.html
+- doc/rdoc/files/lib/pho/upload_rb.html
 - doc/rdoc/files/lib/pho/field_predicate_map_rb.html
 - doc/rdoc/files/lib/pho/job_rb.html
 - doc/rdoc/files/lib/pho/command_line_rb.html
@@ -149,8 +191,12 @@ files:
 - doc/rdoc/classes/Pho/Update/ResourceStatement.html
 - doc/rdoc/classes/Pho/Store.html
 - doc/rdoc/classes/Pho/FileManagement/RDFManager.html
+- doc/rdoc/classes/Pho/FileManagement/BNodeRewritingHandler.html
 - doc/rdoc/classes/Pho/FileManagement/AbstractFileManager.html
+- doc/rdoc/classes/Pho/FileManagement/StatementHandler.html
 - doc/rdoc/classes/Pho/FileManagement/FileManager.html
+- doc/rdoc/classes/Pho/FileManagement/FileSplitter.html
+- doc/rdoc/classes/Pho/FileManagement/Util.html
 - doc/rdoc/classes/Pho/QueryProfile.html
 - doc/rdoc/classes/Pho/JobUpdate.html
 - doc/rdoc/classes/Pho/ResourceHash.html
@@ -185,9 +231,11 @@ files:
 - tests/tc_facet.rb
 - tests/tc_metabox.rb
 - tests/tc_jobcontrol.rb
+- tests/tc_filesplitter.rb
 - tests/ts_pho.rb
 - tests/tc_contentbox.rb
 - tests/tc_changeset_builder.rb
+- tests/tc_bnodehandler.rb
 - tests/tc_rdf_collection.rb
 - tests/tc_converter.rb
 - tests/tc_field_predicate_map.rb
@@ -222,11 +270,11 @@ files:
 - lib/pho/changeset.rb
 - lib/pho/query_profile.rb
 - lib/pho/facet.rb
+- lib/pho/upload.rb
 - lib/pho/enrichment.rb
 - lib/pho/etags.rb
 - lib/pho/field_predicate_map.rb
 - lib/pho.rb
-has_rdoc: true
 homepage: http://pho.rubyforge.net
 licenses: []
 
@@ -262,7 +310,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: pho
-rubygems_version: 1.3.7
+rubygems_version: 1.8.9
 signing_key: 
 specification_version: 3
 summary: Ruby client for the Talis Platform