filecluster 0.3.0 → 0.3.1

data/README.md

@@ -40,12 +40,14 @@ Selecting available storage to copy item by storage.copy_storages (from left to
 |global_daemon_host||set fc_daemon when run global daemon task (only in one instance)|
 |daemon_cycle_time|30|time between global daemon checks and storages available checks|
 |daemon_global_wait_time|120|time between runs global daemon if it does not running|
-|daemon_global_tasks_group_limit|1000|limit for select for tasks|
+|daemon_tasks_copy_group_limit|1000|select limit for copy tasks|
+|daemon_tasks_delete_group_limit|10000|select limit for delete tasks|
+|daemon_tasks_copy_threads_limit|10|copy tasks threads count limit for one storage|
+|daemon_tasks_delete_threads_limit|10|delete tasks threads count limit for one storage|
+|daemon_copy_tasks_per_host_limit|10|copy tasks count limit for one host|
+|daemon_global_tasks_group_limit|1000|select limit for create copy tasks|
 |daemon_global_error_items_ttl|86400|ttl for items with error status before delete|
 |daemon_global_error_items_storages_ttl|86400|ttl for items_storages with error status before delete|
-|daemon_global_tasks_per_thread|10|tasks count for one task thread|
-|daemon_global_tasks_threads_limit|10|tasks threads count limit for one storage|
-|daemon_copy_tasks_limit|10|copy tasks count limit for one host|
 |daemon_restart_period|86400|time between fc-daemon self restart|
 
 ## Usage

data/bin/fc-daemon

@@ -8,16 +8,19 @@ require 'filecluster'
 require 'utils'
 require 'daemon'
 
-$storages       = []          # storages on current host
-$tasks          = {}          # tasks by storage name
-$curr_tasks     = []          # current tasks
-$tasks_threads  = {}          # threads by storage name
-$check_threads  = {}          # threads by storage name
-$copy_count     = 0           # copy tasks count
-$exit_signal    = false 
+$all_storages         = []          # all storages
+$storages             = []          # storages on current host
+$tasks_copy           = {}          # copy tasks by storage name
+$tasks_delete         = {}          # delete tasks by storage name
+$curr_tasks           = []          # current tasks
+$tasks_copy_threads   = {}          # copy threads by storage name
+$tasks_delete_threads = {}          # delete threads by storage name
+$check_threads        = {}          # check threads by storage name
+$copy_count           = 0           # copy tasks count for current host
+$exit_signal          = false 
 $global_daemon_thread = nil
-$update_tasks_thread = nil
-$run_tasks_thread = nil
+$update_tasks_thread  = nil
+$run_tasks_thread     = nil
 
 default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'
 descriptions = {

data/lib/daemon.rb

@@ -4,7 +4,8 @@ require "daemon/check_thread"
 require "daemon/global_daemon_thread"
 require "daemon/run_tasks_thread"
 require "daemon/update_tasks_thread"
-require "daemon/task_thread"
+require "daemon/copy_task_thread"
+require "daemon/delete_task_thread"
 
 def error(msg, options = {})
   $log.error(msg)

data/lib/daemon/copy_task_thread.rb

@@ -0,0 +1,45 @@
+class CopyTaskThread < BaseThread
+  def go(storage_name)
+    return unless $tasks_copy[storage_name]
+    Thread.current[:tasks_processed] = 0 unless Thread.current[:tasks_processed]
+    while task = $tasks_copy[storage_name].shift do
+      $curr_tasks << task
+      $log.debug("CopyTaskThread(#{storage_name}): run task for item_storage ##{task.id}")
+      make_copy(task)
+      $curr_tasks.delete(task)
+      $log.debug("CopyTaskThread(#{storage_name}): finish task for item_storage ##{task.id}")
+      Thread.current[:tasks_processed] += 1
+      exit if $exit_signal
+    end
+  end
+  
+  def make_copy(task)
+    sleep 0.1 while $copy_count > FC::Var.get('daemon_copy_tasks_per_host_limit', 10).to_i
+    $copy_count += 1
+    storage = $storages.detect{|s| s.name == task.storage_name}
+    begin
+      item = FC::Item.find(task.item_id)
+    rescue Exception => e
+      if e.message.match('Record not found')
+        $log.warn("Item ##{task.item_id} not found before copy")
+        return nil
+      else 
+        raise e
+      end
+    end
+    return nil unless item && item.status == 'ready'
+    src_item_storage = FC::ItemStorage.where("item_id = ? AND status = 'ready'", item.id).sample
+    unless src_item_storage
+      $log.warn("Item ##{item.id} #{item.name} has no ready item_storage")
+      return nil 
+    end
+    src_storage = $all_storages.detect{|s| s.name == src_item_storage.storage_name}
+    $log.debug("Copy from #{src_storage.name} to #{storage.name} #{storage.path}#{item.name}")
+    item.copy_item_storage(src_storage, storage, task)
+  rescue Exception => e
+    error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => task.item_id, :item_storage_id => task.id
+    $curr_tasks.delete(task)
+  ensure 
+    $copy_count -= 1 if $copy_count > 0
+  end
+end

data/lib/daemon/delete_task_thread.rb

@@ -0,0 +1,35 @@
+class DeleteTaskThread < BaseThread
+  def go(storage_name)
+    return unless $tasks_delete[storage_name]
+    Thread.current[:tasks_processed] = 0 unless Thread.current[:tasks_processed]
+    while task = $tasks_delete[storage_name].shift do
+      $curr_tasks << task
+      $log.debug("DeleteTaskThread(#{storage_name}): run task for item_storage ##{task.id}")
+      make_delete(task)
+      $curr_tasks.delete(task)
+      $log.debug("DeleteTaskThread(#{storage_name}): finish task for item_storage ##{task.id}")
+      Thread.current[:tasks_processed] += 1
+      exit if $exit_signal
+    end
+  end
+  
+  def make_delete(task)
+    storage = $storages.detect{|s| s.name == task.storage_name}
+    begin
+      item = FC::Item.find(task.item_id)
+    rescue Exception => e
+      if e.message.match('Record not found')
+        $log.warn("Item ##{task.item_id} not found before delete")
+        return nil
+      else 
+        raise e
+      end
+    end
+    $log.debug("Delete #{storage.path}#{item.name}")
+    storage.delete_file(item.name)
+    task.delete
+  rescue Exception => e
+    error "Delete item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => task.item_id, :item_storage_id => task.id
+    $curr_tasks.delete(task)
+  end
+end

data/lib/daemon/run_tasks_thread.rb

@@ -1,24 +1,32 @@
 class RunTasksThread < BaseThread
   def go
-    $log.debug("Run tasks (copy_count: #{$copy_count})")
+    $log.debug('Run tasks')
+    run_tasks(:delete)
+    run_tasks(:copy)
+  end
+  
+  def run_tasks(type)
+    tasks = (type == :copy ? $tasks_copy : $tasks_delete)
+    tasks_threads = (type == :copy ? $tasks_copy_threads : $tasks_delete_threads)
     $storages.each do |storage|
-      $tasks_threads[storage.name] = [] unless $tasks_threads[storage.name]
-      $tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
-      tasks_count = $tasks[storage.name] ? $tasks[storage.name].size : 0
-      threads_count = $tasks_threads[storage.name].count
-      
-      # <max_threads> tasks per thread, maximum <tasks_per_thread> threads
-      max_threads = FC::Var.get('daemon_global_tasks_threads_limit', 10).to_i
-      tasks_per_thread = FC::Var.get('daemon_global_tasks_per_thread', 10).to_i
+      tasks_threads[storage.name] = [] unless tasks_threads[storage.name]
+      tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
+      tasks_count = tasks[storage.name] ? tasks[storage.name].size : 0
+      threads_count = tasks_threads[storage.name].count
+      thread_avg_performance = 0
+      thread_avg_performance = tasks_threads[storage.name].inject(0){|sum, thread| sum+thread[:tasks_processed]} / threads_count if threads_count > 0
+      thread_avg_performance = 10 if thread_avg_performance < 10
+      max_threads = FC::Var.get("daemon_tasks_#{type}_threads_limit", 10).to_i
       
-      run_threads_count = (tasks_count/tasks_per_thread.to_f).ceil
+      run_threads_count = (tasks_count/thread_avg_performance.to_f).ceil
       run_threads_count = max_threads if run_threads_count > max_threads
       run_threads_count = run_threads_count - threads_count
+      run_threads_count = 0 if run_threads_count < 0
       
-      $log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
+      $log.debug("#{storage.name} #{type} tasks_count: #{tasks_count}, threads_count: #{threads_count}, avg_performance: #{thread_avg_performance}, run_threads_count: #{run_threads_count}")
+      tasks_threads[storage.name].each{|thread| thread[:tasks_processed] = 0}
       run_threads_count.times do
-        $log.debug("spawn TaskThread for #{storage.name}") 
-        $tasks_threads[storage.name] << TaskThread.new(storage.name)
+        tasks_threads[storage.name] << (type == :copy ? CopyTaskThread.new(storage.name) : DeleteTaskThread.new(storage.name))
       end
     end
   end

data/lib/daemon/update_tasks_thread.rb

@@ -6,22 +6,21 @@ class UpdateTasksThread < BaseThread
   end
   
   def check_tasks(type)
-    storages_names = $storages.map{|storage| "'#{storage.name}'"}.join(',')
-    return if storages_names.empty?
-    cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
-    ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
-      flatten.map{|task| task[:item_storage].id}
-    ids += $curr_tasks.select{|task| task[:action] == type}.map{|task| task[:item_storage].id}
-    
-    limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
-    cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
-    cond << " LIMIT #{limit}"
-    FC::ItemStorage.where(cond).each do |item_storage|
-      unless ids.include?(item_storage.id)
-        $tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
-        $tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage} 
+    count = 0
+    tasks = (type == :copy ? $tasks_copy : $tasks_delete) 
+    $storages.each do |storage|
+      tasks[storage.name] = [] unless tasks[storage.name]
+      cond = "storage_name = '#{storage.name}' AND status='#{type.to_s}'"
+      ids = tasks[storage.name].map(&:id) + $curr_tasks.map(&:id)
+      limit = FC::Var.get("daemon_tasks_#{type}_group_limit", 1000).to_i
+      cond << "AND id not in (#{ids.join(',')})" if ids.length > 0
+      cond << " LIMIT #{limit}"
+      FC::ItemStorage.where(cond).each do |item_storage|
+        tasks[storage.name] << item_storage 
         $log.debug("task add: type=#{type}, item_storage=#{item_storage.id}")
+        count +=1 
       end
     end
+    $log.debug("Add #{count} #{type} tasks")
   end
 end

data/lib/fc/db.rb

@@ -191,12 +191,14 @@ module FC
       FC::DB.connect.query("CREATE TRIGGER fc_vars_before_update BEFORE UPDATE on #{@prefix}vars FOR EACH ROW BEGIN #{proc_time} END")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_cycle_time', val='30', descr='time between global daemon checks and storages available checks'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_wait_time', val='120', descr='time between runs global daemon if it does not running'")
-      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_group_limit', val='1000', descr='limit for select for tasks'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_tasks_copy_group_limit', val='1000', descr='select limit for copy tasks'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_tasks_delete_group_limit', val='10000', descr='select limit for delete tasks'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_tasks_copy_threads_limit', val='10', descr='copy tasks threads count limit for one storage'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_tasks_delete_threads_limit', val='10', descr='delete tasks threads count limit for one storage'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_per_host_limit', val='10', descr='copy tasks count limit for one host'")
+      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_group_limit', val='1000', descr='select limit for create copy tasks'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_ttl', val='86400', descr='ttl for items with error status before delete'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_error_items_storages_ttl', val='86400', descr='ttl for items_storages with error status before delete'")
-      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_per_thread', val='10', descr='tasks count for one task thread'")
-      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_global_tasks_threads_limit', val='10', descr='tasks threads count limit for one storage'")
-      FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_copy_tasks_limit', val='10', descr='copy tasks count limit for one host'")
       FC::DB.connect.query("INSERT INTO #{@prefix}vars SET name='daemon_restart_period', val='86400', descr='time between fc-daemon self restart'")
     end
   end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/lib/manage/storages.rb

@@ -128,7 +128,7 @@ def storages_change
     storage.url = url unless url.empty?
     storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
     storages = FC::Storage.where.map(&:name)
-    copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
+    storage.copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
     
     puts %Q{\nStorage
     Name:          #{storage.name}
@@ -137,7 +137,7 @@ def storages_change
     Url:           #{storage.url}
     Size:          #{size_to_human storage.size}
     Size limit:    #{size_to_human storage.size_limit}
-    Copy storages: #{copy_storages}}
+    Copy storages: #{storage.copy_storages}}
     s = Readline.readline("Continue? (y/n) ", false).strip.downcase
     puts ""
     if s == "y" || s == "yes"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-11 00:00:00.000000000 Z
+date: 2013-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -146,9 +146,10 @@ files:
 - lib/daemon.rb
 - lib/daemon/base_thread.rb
 - lib/daemon/check_thread.rb
+- lib/daemon/copy_task_thread.rb
+- lib/daemon/delete_task_thread.rb
 - lib/daemon/global_daemon_thread.rb
 - lib/daemon/run_tasks_thread.rb
-- lib/daemon/task_thread.rb
 - lib/daemon/update_tasks_thread.rb
 - lib/fc/base.rb
 - lib/fc/db.rb
@@ -192,7 +193,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2037414088552914900
+      hash: -4221257389535843985
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -201,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2037414088552914900
+      hash: -4221257389535843985
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24

data/lib/daemon/task_thread.rb

@@ -1,53 +0,0 @@
-class TaskThread < BaseThread
-  def go(storage_name)
-    return unless $tasks[storage_name]
-    while task = $tasks[storage_name].shift do
-      $curr_tasks << task
-      $log.debug("TaskThread(#{storage_name}): run task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
-      if task[:action] == :delete
-        make_delete(task)
-      elsif task[:action] == :copy
-        make_copy(task)
-      else
-        error "Unknown task action: #{task[:action]}"
-      end
-      $curr_tasks.delete(task)
-      $log.debug("TaskThread(#{storage_name}): Finish task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
-      exit if $exit_signal
-    end
-  end
-  
-  def make_delete(task)
-    item_storage = task[:item_storage]
-    storage = $storages.detect{|s| s.name == item_storage.storage_name}
-    item = FC::Item.find(item_storage.item_id)
-    $log.debug("Delete #{storage.path}#{item.name}")
-    storage.delete_file(item.name)
-    item_storage.delete
-  rescue Exception => e
-    error "Delete item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
-    $curr_tasks.delete(task)
-  end
-  
-  def make_copy(task)
-    sleep 0.1 while $copy_count > FC::Var.get('daemon_copy_tasks_limit', 10).to_i
-    $copy_count += 1
-    item_storage = task[:item_storage]
-    storage = $storages.detect{|s| s.name == item_storage.storage_name}
-    item = FC::Item.find(item_storage.item_id)
-    return nil unless item && item.status == 'ready'
-    src_item_storage = FC::ItemStorage.where("item_id = ? AND status = 'ready'", item.id).sample
-    unless src_item_storage
-      $log.info("Item ##{item.id} #{item.name} has no ready item_storage")
-      return nil 
-    end
-    src_storage = $all_storages.detect{|s| s.name == src_item_storage.storage_name}
-    $log.debug("Copy from #{src_storage.name} to #{storage.name} #{storage.path}#{item.name}")
-    item.copy_item_storage(src_storage, storage, item_storage)
-  rescue Exception => e
-    error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
-    $curr_tasks.delete(task)
-  ensure 
-    $copy_count -= 1 if item_storage && $copy_count > 0
-  end
-end