filecluster 0.2.3 → 0.2.4

data/README.md

@@ -44,8 +44,8 @@ Selecting available storage to copy item by policy.copy_storages (from left to t
 |daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
 |daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
 |daemon_global_tasks_per_thread|10|tasks count for one task thread|
-|daemon_global_tasks_threads_limit|5|tasks threads count limit for one storage|
-|daemon_copy_tasks_limit|5|copy tasks count limit for one host|
+|daemon_global_tasks_threads_limit|10|tasks threads count limit for one storage|
+|daemon_copy_tasks_limit|10|copy tasks count limit for one host|
 
 ## Usage
 

data/bin/fc-daemon

@@ -16,6 +16,8 @@ $check_threads  = {}          # threads by storage name
 $copy_count     = 0           # copy tasks count
 $exit_signal    = false 
 $global_daemon_thread = nil
+$update_tasks_thread = nil
+$run_tasks_thread = nil
 
 default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
 descriptions = {

data/lib/daemon.rb

@@ -1,7 +1,9 @@
 require "date"
 require "daemon/base_thread"
-require "daemon/global_daemon_thread"
 require "daemon/check_thread"
+require "daemon/global_daemon_thread"
+require "daemon/run_tasks_thread"
+require "daemon/update_tasks_thread"
 require "daemon/task_thread"
 
 def error(msg, options = {})
@@ -57,53 +59,15 @@ def storages_check
 end
 
 def update_tasks
-  $log.debug('Update tasks')
-  return if $storages.length == 0
-  
-  def check_tasks(type)
-    storages_names = $storages.map{|storage| "'#{storage.name}'"}.join(',')
-    cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
-    ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
-      flatten.map{|task| task[:item_storage].id}
-    ids += $curr_tasks.select{|task| task[:action] == type}.map{|task| task[:item_storage].id}
-    
-    limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
-    cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
-    cond << " LIMIT #{limit}"
-    FC::ItemStorage.where(cond).each do |item_storage|
-      unless ids.include?(item_storage.id)
-        $tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
-        $tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage} 
-        $log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
-      end
-    end
+  if !$update_tasks_thread || !$update_tasks_thread.alive?
+    $log.debug("spawn UpdateTasksThread")
+    $update_tasks_thread = UpdateTasksThread.new
   end
-  
-  check_tasks(:delete)
-  check_tasks(:copy)
 end
 
 def run_tasks
-  $log.debug('Run tasks')
-  $storages.each do |storage|
-    $tasks_threads[storage.name] = [] unless $tasks_threads[storage.name]
-    $tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
-    tasks_count = $tasks[storage.name] ? $tasks[storage.name].size : 0
-    threads_count = $tasks_threads[storage.name].count
-    
-    # <max_threads> tasks per thread, maximum <tasks_per_thread> threads
-    max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 5).to_i
-    tasks_per_thread = FC::Var.get('daemon_global_tasks_per_thread', 10).to_i
-    
-    run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil
-    run_threads_count = max_threads if run_threads_count > max_threads
-    run_threads_count = run_threads_count - threads_count
-    
-    $log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
-    $log.debug("copy_count: #{$copy_count}")
-    run_threads_count.times do
-      $log.debug("spawn TaskThread for #{storage.name}") 
-      $tasks_threads[storage.name] << TaskThread.new(storage.name)
-    end
+  if !$run_tasks_thread || !$run_tasks_thread.alive?
+    $log.debug("spawn RunTasksThread")
+    $run_tasks_thread = RunTasksThread.new
   end
 end

data/lib/daemon/run_tasks_thread.rb

@@ -0,0 +1,25 @@
+class RunTasksThread < BaseThread
+  def go
+    $log.debug('Run tasks (copy_count: #{$copy_count})')
+    $storages.each do |storage|
+      $tasks_threads[storage.name] = [] unless $tasks_threads[storage.name]
+      $tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
+      tasks_count = $tasks[storage.name] ? $tasks[storage.name].size : 0
+      threads_count = $tasks_threads[storage.name].count
+      
+      # <max_threads> tasks per thread, maximum <tasks_per_thread> threads
+      max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 10).to_i
+      tasks_per_thread = FC::Var.get('daemon_global_tasks_per_thread', 10).to_i
+      
+      run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil
+      run_threads_count = max_threads if run_threads_count > max_threads
+      run_threads_count = run_threads_count - threads_count
+      
+      $log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
+      run_threads_count.times do
+        $log.debug("spawn TaskThread for #{storage.name}") 
+        $tasks_threads[storage.name] << TaskThread.new(storage.name)
+      end
+    end
+  end
+end

data/lib/daemon/task_thread.rb

@@ -29,8 +29,7 @@ class TaskThread < BaseThread
   end
   
   def make_copy(task)
-    limit = FC::Var.get('daemon_copy_tasks_limit', 10).to_i
-    sleep 0.1 while $copy_count > limit
+    sleep 0.1 while $copy_count > FC::Var.get('daemon_copy_tasks_limit', 10).to_i
     $copy_count += 1
     item_storage = task[:item_storage]
     storage = $storages.detect{|s| s.name == item_storage.storage_name}

data/lib/daemon/update_tasks_thread.rb

@@ -0,0 +1,27 @@
+class UpdateTasksThread < BaseThread
+  def go
+    $log.debug('Update tasks')
+    check_tasks(:delete)
+    check_tasks(:copy)
+  end
+  
+  def check_tasks(type)
+    storages_names = $storages.map{|storage| "'#{storage.name}'"}.join(',')
+    return if storages_names.empty?
+    cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
+    ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
+      flatten.map{|task| task[:item_storage].id}
+    ids += $curr_tasks.select{|task| task[:action] == type}.map{|task| task[:item_storage].id}
+    
+    limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
+    cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
+    cond << " LIMIT #{limit}"
+    FC::ItemStorage.where(cond).each do |item_storage|
+      unless ids.include?(item_storage.id)
+        $tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
+        $tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage} 
+        $log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
+      end
+    end
+  end
+end

data/lib/fc/db.rb

@@ -202,8 +202,8 @@ module FC
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_ttl', val='86400', descr='ttl for items with error status before delete'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
-      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='5', descr='tasks threads count limit for one storage'")
-      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_limit', val='5', descr='copy tasks count limit for one host'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='10', descr='tasks threads count limit for one storage'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_limit', val='10', descr='copy tasks count limit for one host'")
     end
   end
 end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/lib/manage/storages.rb

@@ -120,7 +120,7 @@ def storages_change
     end
     storage.url = url unless url.empty?
     storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
-    storage.copy_id = copy_id.to_i
+    storage.copy_id = copy_id.to_i unless copy_id.empty?
     
     puts %Q{\nStorage
     Name:       #{storage.name}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-16 00:00:00.000000000 Z
+date: 2013-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -147,7 +147,9 @@ files:
 - lib/daemon/base_thread.rb
 - lib/daemon/check_thread.rb
 - lib/daemon/global_daemon_thread.rb
+- lib/daemon/run_tasks_thread.rb
 - lib/daemon/task_thread.rb
+- lib/daemon/update_tasks_thread.rb
 - lib/fc/base.rb
 - lib/fc/db.rb
 - lib/fc/error.rb
@@ -190,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3114920766113706520
+      hash: -4358767098397896012
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -199,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3114920766113706520
+      hash: -4358767098397896012
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24