filecluster 0.2.2 → 0.2.3

data/README.md

@@ -45,6 +45,7 @@ Selecting available storage to copy item by policy.copy_storages (from left to t
 |daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
 |daemon_global_tasks_per_thread|10|tasks count for one task thread|
 |daemon_global_tasks_threads_limit|5|tasks threads count limit for one storage|
+|daemon_copy_tasks_limit|5|copy tasks count limit for one host|
 
 ## Usage
 

data/bin/fc-daemon

@@ -13,6 +13,7 @@ $tasks          = {}          # tasks by storage name
 $curr_tasks     = []          # current tasks
 $tasks_threads  = {}          # threads by storage name
 $check_threads  = {}          # threads by storage name
+$copy_count     = 0           # copy tasks count
 $exit_signal    = false 
 $global_daemon_thread = nil
 

data/lib/daemon.rb

@@ -100,6 +100,7 @@ def run_tasks
     run_threads_count = run_threads_count - threads_count
     
     $log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
+    $log.debug("copy_count: #{$copy_count}")
     run_threads_count.times do
       $log.debug("spawn TaskThread for #{storage.name}") 
       $tasks_threads[storage.name] << TaskThread.new(storage.name)

data/lib/daemon/task_thread.rb

@@ -29,6 +29,9 @@ class TaskThread < BaseThread
   end
   
   def make_copy(task)
+    limit = FC::Var.get('daemon_copy_tasks_limit', 10).to_i
+    sleep 0.1 while $copy_count > limit
+    $copy_count += 1
     item_storage = task[:item_storage]
     storage = $storages.detect{|s| s.name == item_storage.storage_name}
     item = FC::Item.find(item_storage.item_id)
@@ -44,5 +47,7 @@ class TaskThread < BaseThread
   rescue Exception => e
     error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
     $curr_tasks.delete(task)
+  ensure 
+    $copy_count -= 1 if item_storage && $copy_count > 0
   end
 end

data/lib/fc/db.rb

@@ -203,6 +203,7 @@ module FC
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='5', descr='tasks threads count limit for one storage'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_limit', val='5', descr='copy tasks count limit for one host'")
     end
   end
 end

data/lib/fc/policy.rb

@@ -40,17 +40,21 @@ module FC
     # get available storage for copy by copy_id and size
     def get_proper_storage_for_copy(size, copy_id = nil, exclude = [])
       storages = get_copy_storages
-      start_storage_index = nil
-      storages.each_with_index do |s, i|
-        start_storage_index = i if copy_id.to_i == s.copy_id.to_i && !exclude.include?(s.name)
+      storage_index = 0
+      storage_host = nil
+      while s = storages[storage_index]
+        if copy_id.to_i == s.copy_id.to_i && !exclude.include?(s.name)
+          storage_index -= 1 while storage_index > 0 && storages[storage_index-1].host == s.host
+          storage_host = s.host
+          break
+        end
+        storage_index += 1
+      end 
+      storages = (storages[storage_index..-1]+storages[0..storage_index-1]).select do |s|
+        !exclude.include?(s.name) && s.up? && s.size + size < s.size_limit
       end
-      storages = storages[start_storage_index..-1]+storages[0..start_storage_index-1] if storages.size > 0 && start_storage_index
-      storages = storages.select do |storage|
-        !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
-      end
-      storage = storages.detect{|s| copy_id.to_i == s.copy_id.to_i}
-      storage = storages.detect{|s| copy_id.to_i < s.copy_id.to_i} unless storage
-      storage = storages.first unless storage
+      storage = storages.select{|s| storage_host == s.host}.sort{|a,b| b.copy_id.to_i <=> a.copy_id.to_i}.first
+      storage = storages.sort{|a,b| b.copy_id.to_i <=> a.copy_id.to_i}.first unless storage
       storage
     end
   end

data/lib/fc/storage.rb

@@ -84,13 +84,13 @@ module FC
     end
     
     # return object size on storage
-    def file_size(file_name)
+    def file_size(file_name, ignore_errors = false)
       dst_path = "#{self.path}#{file_name}"
       
       cmd = self.class.curr_host == host ? 
         "du -sb #{dst_path}" : 
         "ssh -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} 'du -sb #{dst_path}'"
-      r = `#{cmd} 2>&1`
+      r = ignore_errors ? `#{cmd} 2>/dev/null` : `#{cmd} 2>&1`
       raise r if $?.exitstatus != 0
       r.to_i
     end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/lib/manage/storages.rb

@@ -42,7 +42,7 @@ def storages_add
     path = '/' + path unless path[0] == '/'
     storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_id => copy_id)
     print "Calc current size.. "
-    size = storage.file_size('')
+    size = storage.file_size('', true)
     puts "ok"
   rescue Exception => e
     puts "Error: #{e.message}"
@@ -88,7 +88,7 @@ end
 def storages_update_size
   if storage = find_storage
     print "Calc current size.. "
-    size = storage.file_size('')
+    size = storage.file_size('', true)
     storage.size = size
     begin
       storage.save
@@ -115,7 +115,7 @@ def storages_change
       path = '/' + path unless path[0] == '/'
       storage.path = path 
       print "Calc current size.. "
-      storage.size = storage.file_size('')
+      storage.size = storage.file_size('', true)
       puts "ok"
     end
     storage.url = url unless url.empty?

data/test/policy_test.rb

@@ -6,11 +6,12 @@ class PolicyTest < Test::Unit::TestCase
       @@storages = []
       @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :copy_id => 1, :size_limit => 10)
       @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_id => 2 , :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :copy_id => 4 , :size_limit => 100)
       @@storages.each {|storage| storage.save}
       @@storage3 = FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_id => 3 , :size_limit => 10)
       @@storage3.save
             
-      @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 1')
+      @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 1')
       @@policy.save
     end
     def shutdown
@@ -22,23 +23,23 @@ class PolicyTest < Test::Unit::TestCase
   should "get_create_storages" do
     FC::Policy.storages_cache_time = 10
     assert_same_elements @@storages.map(&:id), @@policy.get_create_storages.map(&:id)
-    @@policy.create_storages = 'rec1-sda,rec2-sda,rec3-sda'
+    @@policy.create_storages = 'rec1-sda,rec2-sda'
     @@policy.save
     assert_equal @@storages.size, @@policy.get_create_storages.size
     FC::Policy.storages_cache_time = 0
-    assert_equal @@storages.size+1, @@policy.get_create_storages.size
+    assert_equal 2, @@policy.get_create_storages.size
   end
   
   should "get_copy_storages" do
-    @@policy.copy_storages = 'rec1-sda,rec2-sda'
+    @@policy.copy_storages = 'rec1-sda,rec2-sda,rec2-sdb'
     @@policy.save
     FC::Policy.storages_cache_time = 10
     assert_same_elements @@storages.map(&:id), @@policy.get_copy_storages.map(&:id)
-    @@policy.copy_storages = 'rec1-sda,rec2-sda,rec3-sda'
+    @@policy.copy_storages = 'rec1-sda,rec2-sda'
     @@policy.save
     assert_equal @@storages.size, @@policy.get_copy_storages.size
     FC::Policy.storages_cache_time = 0
-    assert_equal @@storages.size+1, @@policy.get_copy_storages.size
+    assert_equal 2, @@policy.get_copy_storages.size
   end
   
   should "get_proper_storage_for_create" do
@@ -74,15 +75,19 @@ class PolicyTest < Test::Unit::TestCase
     @@policy.copy_storages = 'rec1-sda,rec3-sda'
     @@policy.save
     assert_equal 'rec3-sda', @@policy.get_proper_storage_for_copy(1, 2).name, 'storage by copy_id'
-    @@policy.copy_storages = 'rec2-sda,rec3-sda,rec1-sda'
+    @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda,rec2-sdb'
     @@policy.save
     assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
+    @@policy.copy_storages = 'rec2-sda,rec3-sda,rec1-sda,rec2-sdb'
+    @@policy.save
+    @@storages[2].update_check_time
+    assert_equal 'rec2-sdb', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
     
-    @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda,'
+    @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda'
     @@policy.save
     @@storages[0].check_time = 0
     @@storages[0].save
-    assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 1).name, 'storage by copy_id'
+    assert_equal 'rec3-sda', @@policy.get_proper_storage_for_copy(1, 1).name, 'storage by copy_id'
   end
   
   should "filter_by_host" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-10 00:00:00.000000000 Z
+date: 2013-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -190,7 +190,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4310476469501781465
+      hash: -3114920766113706520
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -199,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 4310476469501781465
+      hash: -3114920766113706520
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24