pho 0.7.7 → 0.7.8

data/lib/pho/command_line.rb

@@ -149,10 +149,12 @@ module Pho
           resp = @store.store_file( f )  
         end          
       elsif @opts["dir"]
+        if @opts["dir"] = "."
+          @opts["dir"] = File.expand_path(".")
+        end        
         puts "Storing contents of directory: #{@opts["dir"]}"
-        collection = Pho::RDFCollection.new(@store, @opts["dir"])         
-        collection.store()          
-        puts collection.summary()
+        collection = Pho::RDFCollection.new(@store, @opts["dir"])
+        store_collection(collection)         
       else     
         #noop
       end 
@@ -177,15 +179,11 @@ module Pho
         puts "Uploading file: #{ @opts["file"] } to /items/#{ uri } as #{mime}"
         resp = @store.upload_item( f , mime , uri )
       elsif @opts["dir"]
-        puts "Uploading contents of directory: #{@opts["dir"]}"
+        if @opts["dir"] = "."
+          @opts["dir"] = File.expand_path(".")
+        end
         collection = Pho::FileManagement::FileManager.new(@store, @opts["dir"], @opts["base"])
-        if @opts["traverse"]
-          collection.store(:traverse)          
-          puts collection.summary(:traverse)
-        else
-          collection.store()          
-          puts collection.summary()            
-        end         
+        store_collection(collection)
       else     
         #noop
       end 
@@ -200,6 +198,26 @@ module Pho
       end                
     end
     
+    def store_collection(collection)
+      if @opts["force"]
+        puts "Resetting tracking files for directory #{@opts["dir"]}"
+        collection.reset( @opts["traverse"] ? true : false )
+      end
+      if @opts["retry"]
+        puts "Retrying failures in: #{@opts["dir"]}"     
+        collection.retry_failures( @opts["traverse"] ? true : false )
+        puts collection.summary( @opts["traverse"] ? true : false )   
+      elsif @opts["updates"]
+        puts "Uploading updates in: #{@opts["dir"]}"     
+        collection.store_updates( @opts["traverse"] ? true : false )
+        puts collection.summary( @opts["traverse"] ? true : false )           
+      else
+        puts "Uploading contents of directory: #{@opts["dir"]}"
+        collection.store( @opts["traverse"] ? true : false )
+        puts collection.summary( @opts["traverse"] ? true : false )        
+      end              
+    end
+    
     def fpmap(out=$stdout)
       
       if @opts["raw"]

data/lib/pho/file_management.rb

@@ -2,19 +2,18 @@ module Pho
   
   module FileManagement
 
-    #TODO: move files into hidden directory
     class AbstractFileManager
     
       attr_reader :dir
       attr_reader :store
         
-      OK = "ok".freeze
-      FAIL = "fail".freeze
+      OK = "ok"
+      FAIL = "fail"
+      TRACKING_DIR = ".pho"
       
-      def initialize(store, dir, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
+      def initialize(store, dir, ok_suffix=OK, fail_suffix=FAIL)
         @store = store
         @dir = dir
-        @sleep = sleep
         @ok_suffix = ok_suffix
         @fail_suffix = fail_suffix
       end
@@ -29,8 +28,8 @@ module Pho
       end
   
       #Retry anything known to have failed
-      def retry_failures()
-        retries = failures()
+      def retry_failures(recursive=false)
+        retries = failures(recursive)
         retries.each do |filename|
           File.delete( get_fail_file_for(filename) )
           #store it
@@ -38,28 +37,85 @@ module Pho
           store_file(file, filename)      
         end
       end
-            
+     
+      def store_changes(recursive=false)
+        changes = changed_files(recursive)
+        changes.each do |filename|
+          #cleanup tracking files
+          f = get_fail_file_for(filename)
+          File.delete( f ) if File.exists?(f)
+          f = get_ok_file_for(filename)
+          File.delete( f ) if File.exists?(f)
+          file = File.new(filename)
+          store_file(file, filename)
+        end
+      end
+             
       #Reset the directory to clear out any previous statuses
-      def reset()
-        Dir.glob( File.join(@dir, "*.#{@fail_suffix}") ).each do |file|
+      def reset(recursive=false)
+        if recursive
+          pattern = "**/#{TRACKING_DIR}/*"
+        else
+          pattern = "/#{TRACKING_DIR}/*"
+        end        
+        Dir.glob( File.join(@dir, "#{pattern}.#{@fail_suffix}") ).each do |file|
           File.delete(file)
         end
-        Dir.glob( File.join(@dir, "*.#{@ok_suffix}") ).each do |file|
+        Dir.glob( File.join(@dir, "#{pattern}.#{@ok_suffix}") ).each do |file|
           File.delete(file)
         end         
       end
-           
+      
+      #returns true if there is a fail or ok file, false otherwise
+      def stored?(file)
+        ok_file = get_ok_file_for(file)
+        fail_file = get_fail_file_for(file)        
+        if ( File.exists?(ok_file) or File.exists?(fail_file) )
+          return true
+        end        
+        return false
+      end     
+      
+      #Has this file changed (i.e have a newer mtime) than its tracking file
+      #Allows files that have been updated in a collection to be easily identified
+      #New files (i.e untracked) are always considered to be changed
+      def changed?(file)
+        if !stored?(file)
+          return true
+        end
+        ok_file = get_ok_file_for(file)
+        fail_file = get_fail_file_for(file)
+        f = File.new(file)        
+        if ( 
+            ( File.exists?(ok_file) && f.stat.mtime > File.new(ok_file).stat.mtime ) or
+            ( File.exists?(fail_file) && f.stat.mtime > File.new(fail_file).stat.mtime )
+           )
+          return true
+        end        
+        return false        
+      end
+      
+      def stored_files(recursive=false)
+        stored = Array.new
+        list(recursive).each do |file|
+          stored << file if stored?(file)
+        end
+        return stored
+      end
+      
+      def changed_files(recursive=false)
+        changed = Array.new
+        list(recursive).each do |file|
+          changed << file if changed?(file)
+        end
+        return changed
+      end
+            
       #List any new files in the directory
       def new_files(recursive=false)
         newfiles = Array.new
         list(recursive).each do |file|
-          
-          ok_file = get_ok_file_for(file)
-          fail_file = get_fail_file_for(file)
-          if !( File.exists?(ok_file) or File.exists?(fail_file) )
-            newfiles << file          
-          end
-            
+          newfiles << file if !stored?(file)            
         end
         return newfiles
       end
@@ -68,9 +124,7 @@ module Pho
       def failures(recursive=false)
         fails = Array.new
         list(recursive).each do |file|
-          if File.extname(file) != ".#{@fail_suffix}" && File.extname(file) != ".#{@ok_suffix}"
-            fails << file if File.exists?( get_fail_file_for(file) )  
-          end          
+          fails << file if File.exists?( get_fail_file_for(file) )
         end
         return fails
       end
@@ -79,9 +133,7 @@ module Pho
       def successes(recursive=false)
         successes = Array.new
         list(recursive).each do |file|
-          if File.extname(file) != ".#{@fail_suffix}" && File.extname(file) != ".#{@ok_suffix}"
-             successes << file if File.exists?( get_ok_file_for(file) )
-          end
+          successes << file if File.exists?( get_ok_file_for(file) )
         end
         return successes
       end
@@ -98,13 +150,17 @@ module Pho
       end
       
       def get_fail_file_for(filename)
-        ext = File.extname(filename)
-        return filename.gsub(/#{ext}$/, ".#{@fail_suffix}")              
+        relative_path = filename.gsub(@dir, "")
+        base = File.basename(filename)
+        relative_path = relative_path.gsub(base, "#{TRACKING_DIR}/#{base}")
+        return "#{@dir}#{relative_path}.#{@fail_suffix}"              
       end
       
       def get_ok_file_for(filename)
-        ext = File.extname(filename)        
-        return filename.gsub(/#{ext}$/, ".#{@ok_suffix}")
+        relative_path = filename.gsub(@dir, "")
+        base = File.basename(filename)
+        relative_path = relative_path.gsub(base, "#{TRACKING_DIR}/#{base}")
+        return "#{@dir}#{relative_path}.#{@ok_suffix}"              
       end
                  
     end

data/lib/pho/file_manager.rb

@@ -8,25 +8,19 @@ module Pho
    
       attr_reader :base
       
-      def initialize(store, dir, base = nil, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
-        super(store, dir, ok_suffix, fail_suffix, sleep)
+      def initialize(store, dir, base = nil, ok_suffix=OK, fail_suffix=FAIL)
+        super(store, dir, ok_suffix, fail_suffix)
         @base = base
       end
 
       #List files being managed, i.e. everything not .ok or .fail
       def list(recursive=false)          
-          files = []
           if recursive
-            pattern = "**/*"
+            pattern = "**/*.*"
           else
-            pattern = "*"
+            pattern = "*.*"
           end
-          Dir.glob( File.join(@dir, pattern) ) do |file|
-            if File.extname(file) != ".#{@ok_suffix}" && File.extname(file) != ".#{@fail_suffix}" && !File.directory?(file)
-              files << file
-            end
-          end  
-          return files
+          return Dir.glob( File.join(@dir, pattern) )
       end
            
       def FileManager.name_for_file(dir, file, base=nil)
@@ -40,6 +34,7 @@ module Pho
       def store_file(file, filename)
         uri = FileManager.name_for_file(@dir, file, @base)
         response = @store.upload_item(file, MIME::Types.type_for(filename)[0].to_s, uri )
+        create_tracking_dir(filename)            
         if (response.status < 300 )
           File.open(get_ok_file_for(filename), "w") do |file|
             file.print( "OK" )            
@@ -51,6 +46,10 @@ module Pho
         end      
       end  
          
+      def create_tracking_dir(filename)
+        path = filename.split("/")[0..-2].join("/")  
+        Dir.mkdir("#{path}/#{TRACKING_DIR}") unless File.exists?("#{path}/#{TRACKING_DIR}")
+      end
     end
     
   end

data/lib/pho/rdf_collection.rb

@@ -14,14 +14,19 @@ module Pho
         
         ALL_RDF = [RDF, TTL, NT]
         
-        def initialize(store, dir, rdf_suffixes=ALL_RDF, ok_suffix=OK, fail_suffix=FAIL, sleep=1)
-          super(store, dir, ok_suffix, fail_suffix, sleep)
+        def initialize(store, dir, rdf_suffixes=ALL_RDF, ok_suffix=OK, fail_suffix=FAIL)
+          super(store, dir, ok_suffix, fail_suffix)
           @rdf_suffixes = rdf_suffixes
         end
                     
         #List files being managed
         def list(recursive=false)
-            return Dir.glob( File.join(@dir, "*.\{#{ @rdf_suffixes.join(",") }\}") )  
+            if recursive
+              pattern = "**/*.\{#{ @rdf_suffixes.join(",") }\}"
+            else
+              pattern = "*.\{#{ @rdf_suffixes.join(",") }\}"
+            end          
+            return Dir.glob( File.join(@dir, pattern) )  
         end
          
         protected

data/tests/tc_file_manager.rb

@@ -6,33 +6,40 @@ require 'mocha'
 class FileManagerTest < Test::Unit::TestCase
 
   def setup()
-    Dir.mkdir("/tmp/pho") unless File.exists?("/tmp/pho")
-    Dir.mkdir("/tmp/pho/a") unless File.exists?("/tmp/pho/a")
-    Dir.mkdir("/tmp/pho/b") unless File.exists?("/tmp/pho/b")
+    Dir.mkdir("/tmp/pho") unless File.exists?("/tmp/pho")    
+    Dir.mkdir("/tmp/pho/.pho") unless File.exists?("/tmp/pho/.pho")
+
+    Dir.mkdir("/tmp/pho/a") unless File.exists?("/tmp/pho/a")    
+    Dir.mkdir("/tmp/pho/b") unless File.exists?("/tmp/pho/b")        
+
+    #/tmp/pho/[0-6].css
     7.times do |i|
       file = File.new( File.join("/tmp/pho", "#{i}.css"), "w" )
       file.write("CSS#{i}")
       file.close()
     end
+    #/tmp/pho/[7-9].js
     3.times do |i|
       num = i + 7
       file = File.new( File.join("/tmp/pho", "#{num}.js"), "w" )
       file.write("JS#{num}")
       file.close()
     end
+    #/tmp/pho/.pho/[0-3].ok
     4.times do |i|
-      file = File.new( File.join("/tmp/pho", "#{i}.ok"), "w" )
+      file = File.new( File.join("/tmp/pho/.pho", "#{i}.css.ok"), "w" )
       file.write("OK")
       file.close()      
     end
+    #/tmp/pho/.pho/[4-6].fail
     3.times do |i|
       num = 4 + i
-      file = File.new( File.join("/tmp/pho", "#{num}.fail"), "w" )
+      file = File.new( File.join("/tmp/pho/.pho", "#{num}.css.fail"), "w" )
       file.write("FAIL")
       file.close()      
     end
 
-    #/tmp/pho/a
+    #/tmp/pho/a/[0-1].txt
     2.times do |i|
       num = i
       file = File.new( File.join("/tmp/pho/a", "#{num}.txt"), "w" )
@@ -40,7 +47,7 @@ class FileManagerTest < Test::Unit::TestCase
       file.close()
     end
 
-    #/tmp/pho/b
+    #/tmp/pho/b/0.txt
     1.times do |i|
       num = i
       file = File.new( File.join("/tmp/pho/b", "#{num}.txt"), "w" )
@@ -51,23 +58,101 @@ class FileManagerTest < Test::Unit::TestCase
   end
   
   def teardown()
-    Dir.glob("/tmp/pho/*.css") do |file|
+    Dir.glob("/tmp/pho/*.*") do |file|
       File.delete(file)
     end
-    Dir.glob("/tmp/pho/*.js") do |file|
+    Dir.glob("/tmp/pho/.pho/*.*") do |file|
       File.delete(file)
-    end    
-    Dir.glob("/tmp/pho/**/*.ok") do |file|
+    end
+    Dir.glob("/tmp/pho/a/.pho/*.*") do |file|
       File.delete(file)
     end
-    Dir.glob("/tmp/pho/**/*.fail") do |file|
+    Dir.glob("/tmp/pho/b/.pho/*.*") do |file|
       File.delete(file)
     end
-    Dir.glob("/tmp/pho/**/*.txt") do |file|
+    Dir.glob("/tmp/pho/a/*.*") do |file|
       File.delete(file)
-    end                    
+    end
+    Dir.glob("/tmp/pho/b/*.*") do |file|
+      File.delete(file)
+    end
+    
+    delete("/tmp/pho/a/.pho")
+    delete("/tmp/pho/b/.pho")
+    delete("/tmp/pho/a")
+    delete("/tmp/pho/b")
+    delete("/tmp/pho/.pho")    
+    delete("/tmp/pho")                        
   end
       
+  def delete(dir)
+    Dir.delete(dir) if File.exists?(dir)
+  end
+  
+  def test_get_ok_file_for()
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")    
+    collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
+    assert_equal( "/tmp/pho/.pho/0.css.ok", collection.get_ok_file_for("/tmp/pho/0.css") )
+    assert_equal( "/tmp/pho/a/.pho/0.txt.ok", collection.get_ok_file_for("/tmp/pho/a/0.txt") )    
+  end
+
+  def test_get_fail_file_for()    
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")    
+    collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
+    assert_equal( "/tmp/pho/.pho/0.css.fail", collection.get_fail_file_for("/tmp/pho/0.css") )
+    assert_equal( "/tmp/pho/a/.pho/0.txt.fail", collection.get_fail_file_for("/tmp/pho/a/0.txt") )    
+  end
+  
+  def test_stored()
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")    
+    collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
+    7.times do |i|
+      assert_equal( true, collection.stored?("/tmp/pho/#{i}.css"), "#{i}.css should be stored" )
+    end
+        
+  end
+      
+  def test_changed()
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")    
+    collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
+    7.times do |i|
+      assert_equal( false, collection.changed?("/tmp/pho/#{i}.css"), "#{i}.css should not be changed" )
+    end    
+    
+    sleep(1)
+    #check we're not creating it with the touch
+    assert_equal( true, File.exists?("/tmp/pho/0.css") )
+    FileUtils.touch( "/tmp/pho/0.css")    
+    assert_equal( true, collection.changed?("/tmp/pho/0.css") )
+    #other files are unchanged
+    6.times do |i|
+      assert_equal( false, collection.changed?("/tmp/pho/#{i+1}.css"), "#{i+1}.css should not be changed" )
+    end    
+    #untracked files are also changed
+    3.times do |i|
+      assert_equal( true, collection.changed?("/tmp/pho/#{i+7}.js"), "#{i+7}.js should not be changed" )
+    end    
+    
+  end
+  
+  def test_changed_files()
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")    
+    collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
+    changed = collection.changed_files()
+    assert_equal( 3, changed.size )
+    
+    sleep(1)
+    #check we're not creating it with the touch
+    assert_equal( true, File.exists?("/tmp/pho/0.css") )
+    FileUtils.touch( "/tmp/pho/0.css")
+        
+    changed = collection.changed_files()
+    changed.sort!
+    assert_equal( 4, changed.size )
+    assert_equal( "/tmp/pho/0.css", changed[0] )
+
+  end
+    
   def test_new_files()
     store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")
     collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
@@ -110,9 +195,9 @@ class FileManagerTest < Test::Unit::TestCase
       collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
       collection.store()
       
-      assert_equal(true, File.exists?("/tmp/pho/7.ok") )      
-      assert_equal(true, File.exists?("/tmp/pho/8.ok") )
-      assert_equal(true, File.exists?("/tmp/pho/9.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/.pho/7.js.ok") )      
+      assert_equal(true, File.exists?("/tmp/pho/.pho/8.js.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/.pho/9.js.ok") )
       
   end
 
@@ -128,9 +213,9 @@ class FileManagerTest < Test::Unit::TestCase
       collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho", "assets")
       collection.store()
       
-      assert_equal(true, File.exists?("/tmp/pho/7.ok") )      
-      assert_equal(true, File.exists?("/tmp/pho/8.ok") )
-      assert_equal(true, File.exists?("/tmp/pho/9.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/.pho/7.js.ok") )      
+      assert_equal(true, File.exists?("/tmp/pho/.pho/8.js.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/.pho/9.js.ok") )
       
   end
 
@@ -150,12 +235,12 @@ class FileManagerTest < Test::Unit::TestCase
       collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
       collection.store(:recursive)
       
-      assert_equal(true, File.exists?("/tmp/pho/7.ok") )      
-      assert_equal(true, File.exists?("/tmp/pho/8.ok") )
-      assert_equal(true, File.exists?("/tmp/pho/9.ok") )
-      assert_equal(true, File.exists?("/tmp/pho/a/0.ok") )
-      assert_equal(true, File.exists?("/tmp/pho/a/1.ok") )
-      assert_equal(true, File.exists?("/tmp/pho/b/0.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/.pho/7.js.ok") )      
+      assert_equal(true, File.exists?("/tmp/pho/.pho/8.js.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/.pho/9.js.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/a/.pho/0.txt.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/a/.pho/1.txt.ok") )
+      assert_equal(true, File.exists?("/tmp/pho/b/.pho/0.txt.ok") )
       
   end
   
@@ -175,4 +260,37 @@ class FileManagerTest < Test::Unit::TestCase
     assert_equal(13, files.size)
   end
   
+  def test_reset()
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")
+    collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
+  
+    collection.reset()
+    newfiles = collection.new_files()
+    assert_equal(10, newfiles.size)        
+  end
+
+  def test_reset_recursive()
+    store = Pho::Store.new("http://api.talis.com/stores/testing", "user", "pass")
+    collection = Pho::FileManagement::FileManager.new(store, "/tmp/pho")
+  
+    Dir.mkdir("/tmp/pho/b/.pho") unless File.exists?("/tmp/pho/b/.pho")
+    
+    file = File.new( File.join("/tmp/pho/b/.pho", "0.txt.fail"), "w" )
+    file.write("FAIL")
+    file.close()      
+    
+    files = collection.failures(:recurse)
+    assert_equal(4, files.size)        
+    assert_equal( true, files.include?("/tmp/pho/b/0.txt") )
+        
+    collection.reset(:recurse)
+    newfiles = collection.new_files(:recurse)
+    assert_equal(13, newfiles.size)        
+    assert_equal( true, newfiles.include?("/tmp/pho/b/0.txt") )
+    
+    files = collection.failures(:recurse)
+    assert_equal(0, files.size)
+    
+    assert_equal( true, !File.exists?("/tmp/pho/b/.pho/0.txt.fail") )
+  end    
 end