filecluster 0.2.1 → 0.2.2

data/README.md

@@ -44,7 +44,7 @@ Selecting available storage to copy item by policy.copy_storages (from left to t
 |daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
 |daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
 |daemon_global_tasks_per_thread|10|tasks count for one task thread|
-|daemon_global_tasks_threads_limit|3|tasks threads count limit for one storage|
+|daemon_global_tasks_threads_limit|5|tasks threads count limit for one storage|
 
 ## Usage
 

data/lib/daemon.rb

@@ -30,11 +30,9 @@ def run_global_daemon
       $log.debug("spawn GlobalDaemonThread")
       $global_daemon_thread = GlobalDaemonThread.new(timeout)
     end 
-  else
-    if $global_daemon_thread
-      $log.warn("Kill global daemon thread (new host = #{r['host']})")
-      $global_daemon_thread.exit
-    end 
+  elsif $global_daemon_thread && $global_daemon_thread.alive?
+    $log.warn("Kill global daemon thread (new host = #{r['val']})")
+    $global_daemon_thread.exit
   end
 end
 
@@ -94,11 +92,13 @@ def run_tasks
     threads_count = $tasks_threads[storage.name].count
     
     # <max_threads> tasks per thread, maximum <tasks_per_thread> threads
-    max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 3).to_i
+    max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 5).to_i
     tasks_per_thread = FC::Var.get('daemon_global_tasks_per_thread', 10).to_i
     
-    run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil - threads_count
+    run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil
     run_threads_count = max_threads if run_threads_count > max_threads
+    run_threads_count = run_threads_count - threads_count
+    
     $log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
     run_threads_count.times do
       $log.debug("spawn TaskThread for #{storage.name}") 

data/lib/daemon/global_daemon_thread.rb

@@ -38,25 +38,28 @@ class GlobalDaemonThread < BaseThread
     end
     
     limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
-    sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name ORDER BY ist.id) as storages, p.id as policy_id, p.copies as policy_copies "+
-      "FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p, #{FC::ItemStorage.table_name} as ist WHERE "+
-      "i.policy_id = p.id AND ist.item_id = i.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT #{limit}"
-    r = FC::DB.query(sql)
-    r.each do |row|
-      $log.info("GlobalDaemonThread: new item_storage for item #{row['item_id']}")
-      item_storages = row['storages'].split(',')
-      if row['item_copies'] != item_storages.size
-        $log.warn("GlobalDaemonThread: ItemStorage count <> item.copies for item #{row['item_id']}")
-      elsif item_storages.size >= row['policy_copies']
-        $log.warn("GlobalDaemonThread: ItemStorage count >= policy.copies for item #{row['item_id']}")
-      else
-        src_storage = all_storages.detect{|s| item_storages.first == s.name}
-        policy = all_policies.detect{|p| row['policy_id'] == p.id}
-        storage = policy.get_proper_storage_for_copy(row['size'], src_storage.copy_id, item_storages) if src_storage && policy 
-        if storage
-          FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
+    all_policies.each do |policy|
+      next if policy.copies.to_i < 2
+      copies = (1..policy.copies.to_i-1).to_a.join(',')
+      sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name ORDER BY ist.id) as storages "+
+        "FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE i.policy_id = #{policy.id} AND "+
+        "ist.item_id = i.id AND i.copies IN (#{copies}) AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT #{limit}"
+      r = FC::DB.query(sql)
+      r.each do |row|
+        $log.info("GlobalDaemonThread: new item_storage for item #{row['item_id']}")
+        item_storages = row['storages'].split(',')
+        if row['item_copies'] != item_storages.count
+          $log.warn("GlobalDaemonThread: ItemStorage count <> item.copies for item #{row['item_id']}")
+        elsif item_storages.count >= policy.copies.to_i
+          $log.warn("GlobalDaemonThread: ItemStorage count >= policy.copies for item #{row['item_id']}")
         else
-          error 'No available storage', :item_id => row['item_id']
+          src_storage = all_storages.detect{|s| item_storages.first == s.name}
+          storage = policy.get_proper_storage_for_copy(row['size'], src_storage.copy_id, item_storages) if src_storage && policy 
+          if storage
+            FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
+          else
+            error 'No available storage', :item_id => row['item_id']
+          end
         end
       end
     end

data/lib/daemon/task_thread.rb

@@ -13,6 +13,7 @@ class TaskThread < BaseThread
       end
       $curr_tasks.delete(task)
       $log.debug("TaskThread(#{storage_name}): Finish task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
+      exit if $exit_signal
     end
   end
   

data/lib/fc/db.rb

@@ -202,7 +202,7 @@ module FC
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_ttl', val='86400', descr='ttl for items with error status before delete'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
-      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='3', descr='tasks threads count limit for one storage'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='5', descr='tasks threads count limit for one storage'")
     end
   end
 end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-04 00:00:00.000000000 Z
+date: 2013-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -190,7 +190,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1102221497557017390
+      hash: 4310476469501781465
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -199,7 +199,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 1102221497557017390
+      hash: 4310476469501781465
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24