filecluster 0.3.12 → 0.4.0

data/lib/fc/db.rb

@@ -43,7 +43,12 @@ module FC
         puts "Deadlock"
         sleep 0.1
         self.query(sql)
-      else 
+      elsif e.message.match('Lost connection to MySQL server during query')
+        puts "Lost connection to MySQL server during query"
+        FC::DB.connect.ping
+        sleep 0.1
+        self.query(sql)
+      else
         raise e
       end
     end

data/lib/fc/item.rb

@@ -26,7 +26,7 @@ module FC
 
       if local_path.include?(item_name) && !options[:not_local]
         storage = policy.get_create_storages.detect do |s|
-          s.host == FC::Storage.curr_host && local_path.index(s.path) == 0 && local_path.sub(s.path, '') == item_params[:name]
+          s.host == FC::Storage.curr_host && local_path.index(s.path) == 0 && local_path.sub(s.path, '').sub(/\/$/, '').sub(/^\//, '') == item_params[:name]
         end
         FC::Error.raise "local_path #{local_path} is not valid path for policy ##{policy.id}" unless storage
       end
@@ -48,8 +48,8 @@ module FC
       if storage
         item_storage = item.make_item_storage(storage, 'ready')
         item.reload
-      else 
-        storage = policy.get_proper_storage_for_create(item.size)
+      else
+        storage = policy.get_proper_storage_for_create(item.size, File.realdirpath(local_path))
         FC::Error.raise 'No available storage', :item_id => item.id unless storage
         
         # mark delete item_storages on replace

data/lib/fc/policy.rb

@@ -35,9 +35,27 @@ module FC
       end
     end
     
-    # get available storage for create by size
-    def get_proper_storage_for_create(size, exclude = [])
-      get_proper_storages_for_create(size, exclude).first
+    # get available storage for create by size and local item path
+    def get_proper_storage_for_create(size, local_path = nil)
+      storages = get_proper_storages_for_create(size)
+      # sort by current_host and free size
+      storages.sort do |a, b|
+        if FC::Storage.curr_host == a.host && FC::Storage.curr_host == b.host
+          if local_path && local_path.index(a.path) == 0
+            1
+          elsif local_path && local_path.index(b.path) == 0
+            -1
+          else
+            a.free_rate <=> b.free_rate
+          end
+        elsif FC::Storage.curr_host == a.host
+          1
+        elsif FC::Storage.curr_host == b.host
+          -1
+        else
+          a.free_rate <=> b.free_rate
+        end
+      end.last
     end
   end
 end

data/lib/fc/storage.rb

@@ -26,6 +26,18 @@ module FC
       super params
     end
     
+    def free
+      size_limit - size
+    end
+    
+    def size_rate
+      size.to_f / size_limit
+    end
+    
+    def free_rate
+      free.to_f / size_limit
+    end
+    
     def get_copy_storages
       self.class.get_copy_storages_mutex.synchronize do
         unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.3.12"
+  VERSION = "0.4.0"
 end

data/lib/manage/storages.rb

@@ -8,6 +8,7 @@ def storages_list
   else
     storages.each do |storage|
       str = "#{colorize_string(storage.host, :yellow)} #{storage.name} #{size_to_human(storage.size)}/#{size_to_human(storage.size_limit)} "
+      str += "#{(storage.free_rate*100).to_i}% free "
       str += "#{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}"
       str += " #{storage.check_time_delay} seconds ago" if storage.check_time
       puts str
@@ -23,7 +24,8 @@ def storages_show
   Host:           #{storage.host} 
   Path:           #{storage.path} 
   Url:            #{storage.url}
-  Size:           #{size_to_human storage.size}
+  Size:           #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
+  Free:           #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%) 
   Size limit:     #{size_to_human storage.size_limit}
   Copy storages:  #{storage.copy_storages}
   Check time:     #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
@@ -53,12 +55,14 @@ def storages_add
     puts "Error: #{e.message}"
     exit
   end
+  free = size_limit - size
   puts %Q{\nStorage
   Name:         #{name}
   Host:         #{host} 
   Path:         #{path} 
   Url:          #{url}
-  Size:         #{size_to_human size}
+  Size:         #{size_to_human size} (#{(size.to_f*100 / size_limit).to_i}%)
+  Free:         #{size_to_human free} (#{(free.to_f*100 / size_limit).to_i}%)
   Size limit:   #{size_to_human size_limit}
   Copy storages #{copy_storages}}
   s = Readline.readline("Continue? (y/n) ", false).strip.downcase
@@ -135,7 +139,8 @@ def storages_change
     Host:          #{storage.host} 
     Path:          #{storage.path} 
     Url:           #{storage.url}
-    Size:          #{size_to_human storage.size}
+    Size:          #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
+    Free:          #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
     Size limit:    #{size_to_human storage.size_limit}
     Copy storages: #{storage.copy_storages}}
     s = Readline.readline("Continue? (y/n) ", false).strip.downcase

data/test/functional_test.rb

@@ -16,11 +16,11 @@ class FunctionalTest < Test::Unit::TestCase
       `cp #{@@test_file_path.shellescape} #{@@test_dir_path.shellescape}/bbb/test2`
       
       @@storages = []
-      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size_limit => 1000000, :check_time => Time.new.to_i)
-      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :size_limit => 1000000, :check_time => Time.new.to_i)
-      @@storages << FC::Storage.new(:name => 'host2-sda', :host => 'host2', :path => '/tmp/host2-sda/', :size_limit => 1000000, :check_time => Time.new.to_i)
-      @@storages << FC::Storage.new(:name => 'host2-sdb', :host => 'host2', :path => '/tmp/host2-sdb/', :size_limit => 1000000, :check_time => Time.new.to_i)
-      @@storages << FC::Storage.new(:name => 'host3-sda', :host => 'host3', :path => '/tmp/host3-sda/', :size_limit => 1000000)
+      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size => 0, :size_limit => 1000000, :check_time => Time.new.to_i)
+      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :size => 0, :size_limit => 1000000, :check_time => Time.new.to_i)
+      @@storages << FC::Storage.new(:name => 'host2-sda', :host => 'host2', :path => '/tmp/host2-sda/', :size => 10, :size_limit => 1000000, :check_time => Time.new.to_i)
+      @@storages << FC::Storage.new(:name => 'host2-sdb', :host => 'host2', :path => '/tmp/host2-sdb/', :size => 10, :size_limit => 1000000, :check_time => Time.new.to_i)
+      @@storages << FC::Storage.new(:name => 'host3-sda', :host => 'host3', :path => '/tmp/host3-sda/', :size => 100, :size_limit => 1000000)
       @@storages.each { |storage| storage.save}
       
       @@policies = []
@@ -113,6 +113,13 @@ class FunctionalTest < Test::Unit::TestCase
   should "item create_from_local inplace" do
     tmp_file_path = "/tmp/host2-sda/inplace test"
     `cp #{@@test_file_path.shellescape} #{tmp_file_path.shellescape}`
-    assert_nothing_raised { @item = FC::Item.create_from_local(tmp_file_path, 'inplace test', @@policies[0]) }
+    assert_nothing_raised { FC::Item.create_from_local(tmp_file_path, 'inplace test', @@policies[0]) }
+  end
+  
+  should "item create_from_local inplace for dir" do
+    tmp_dir_path = "/tmp/host2-sda/inplace test dir/"
+    `mkdir #{tmp_dir_path.shellescape}`
+    `cp #{@@test_file_path.shellescape} #{tmp_dir_path.shellescape}`
+    assert_nothing_raised { FC::Item.create_from_local(tmp_dir_path, '/inplace test dir/', @@policies[0]) }
   end
 end

data/test/policy_test.rb

@@ -4,14 +4,16 @@ class PolicyTest < Test::Unit::TestCase
   class << self
     def startup
       @@storages = []
-      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :size_limit => 10)
-      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :size_limit => 100)
-      @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :path => '/tmp/host1-sda/', :size => 0, :size_limit => 10)
+      @@storages << FC::Storage.new(:name => 'rec1-sdb', :host => 'rec1', :path => '/tmp/host1-sdb/', :size => 0, :size_limit => 10)
+      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :path => '/tmp/host2-sda/', :size => 30, :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :path => '/tmp/host2-sdb/', :size => 20, :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec2-sdc', :host => 'rec2', :path => '/tmp/host2-sdc/', :size => 10, :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :path => '/tmp/host3-sda/', :size => 99, :size_limit => 1000)
+      @@storages << FC::Storage.new(:name => 'rec3-sdb', :host => 'rec3', :path => '/tmp/host3-sdb/', :size => 199, :size_limit => 1000)
       @@storages.each {|storage| storage.save}
-      @@storage3 = FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :size_limit => 10)
-      @@storage3.save
-            
-      @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copies => 1, :name => 'policy 1')
+      
+      @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb,rec2-sdc,rec3-sda,rec3-sdb', :copies => 1, :name => 'policy 1')
       @@policy.save
     end
     def shutdown
@@ -22,32 +24,50 @@ class PolicyTest < Test::Unit::TestCase
 
   should "get_create_storages" do
     FC::Policy.storages_cache_time = 10
-    assert_same_elements @@storages.map(&:id), @@policy.get_create_storages.map(&:id)
+    assert_equal 6, @@policy.get_create_storages.size
     @@policy.create_storages = 'rec1-sda,rec2-sda'
     @@policy.save
-    assert_equal @@storages.size, @@policy.get_create_storages.size
+    assert_equal 6, @@policy.get_create_storages.size
     FC::Policy.storages_cache_time = 0
     assert_equal 2, @@policy.get_create_storages.size
+    @@policy.create_storages = 'rec1-sda,rec2-sda,rec2-sdb,rec2-sdc,rec3-sda,rec3-sdb'
+    @@policy.save
+  end
+  
+  should "filter_by_host" do
+    FC::Storage.stubs(:curr_host).returns('rec2')
+    FC::Policy.new(:create_storages => 'rec3-sda,rec2-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 2').save
+    FC::Policy.new(:create_storages => 'rec1-sda,rec3-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 3').save
+    assert_same_elements ['policy 1', 'policy 2'], FC::Policy.filter_by_host.map{|p| p.name}
   end
   
   should "get_proper_storage_for_create" do
-    @@storages.each {|storage| storage.check_time = 0; storage.save}
-    @@storage3.check_time = 0
-    @@storage3.save
     FC::Policy.storages_cache_time = 0
+    @@storages.each {|storage| storage.check_time = 0; storage.save}
     assert_nil @@policy.get_proper_storage_for_create(1), 'all storages down'
     @@storages[0].update_check_time
-    assert_equal @@storages[0].id, @@policy.get_proper_storage_for_create(5).id, 'first storages up'
+    assert_kind_of FC::Storage, @@policy.get_proper_storage_for_create(5), 'first storages up'
+    assert_equal 'rec1-sda', @@policy.get_proper_storage_for_create(5).name, 'first storages up'
     assert_nil @@policy.get_proper_storage_for_create(20), 'first storage full'
-    @@storages[1].update_check_time
-    assert_equal @@storages[1].id, @@policy.get_proper_storage_for_create(20).id, 'second storages up'
+    @@storages[2].update_check_time
+    assert_equal 'rec2-sda', @@policy.get_proper_storage_for_create(20).name, 'second storages up'
     assert_nil @@policy.get_proper_storage_for_create(1000), 'all storages full'
   end
   
-  should "filter_by_host" do
+  should "get_proper_storage_for_create for local path" do
+    FC::Policy.storages_cache_time = 0
+    @@storages.each {|storage| storage.update_check_time}
     FC::Storage.stubs(:curr_host).returns('rec2')
-    FC::Policy.new(:create_storages => 'rec3-sda,rec2-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 2').save
-    FC::Policy.new(:create_storages => 'rec1-sda,rec3-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 3').save
-    assert_same_elements ['policy 1', 'policy 2'], FC::Policy.filter_by_host.map{|p| p.name}
+    assert_equal 'rec2-sdb', @@policy.get_proper_storage_for_create(9, '/tmp/host2-sdb/test').name, 'current host, local path match'
+    @@storages[3].check_time = 0;
+    @@storages[3].save
+    assert_equal 'rec2-sdc', @@policy.get_proper_storage_for_create(9, '/tmp/host2-sdb/test').name, 'current host, most free storage'
+    FC::Storage.stubs(:curr_host).returns('rec3')
+    @@storages[5].check_time = 0;
+    @@storages[5].save
+    assert_equal 'rec3-sdb', @@policy.get_proper_storage_for_create(9, '/tmp/host3-sdc/test').name, 'current host, single storage'
+    FC::Storage.stubs(:curr_host).returns('rec5')
+    assert_equal 'rec1-sda', @@policy.get_proper_storage_for_create(9, '/tmp/host3-sdc/test').name, 'not current host, most free storage'
+    assert_equal 'rec2-sdc', @@policy.get_proper_storage_for_create(10, '/tmp/host3-sdc/test').name, 'not current host, big file, most free storage with free space'
   end
-end
+end

data/test/storage_test.rb

@@ -69,4 +69,12 @@ class StorageTest < Test::Unit::TestCase
     assert_equal 'rec2-sda', @@storages[2].get_proper_storage_for_copy(20).name, 'second storages up, big file'
     assert_nil @@storages[2].get_proper_storage_for_copy(1000), 'second storages up, huge file'
   end
+  
+  should "free and rate" do
+    storage = @@storages[0]
+    storage.size = 1
+    assert_equal 9, storage.free, 'first storage free must be size_limit - size'
+    assert_equal 9.to_f/10, storage.free_rate, 'first storage free_rate must be free/size_limit' 
+    assert_equal 1.to_f/10, storage.size_rate, 'first storage free must be size/size_limit'
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.3.12
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-18 00:00:00.000000000 Z
+date: 2014-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2
@@ -181,7 +181,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2800763299588068596
+      hash: -1129513891376945174
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -190,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2800763299588068596
+      hash: -1129513891376945174
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24