filecluster 0.1.6 → 0.1.7

data/bin/fc-daemon

@@ -8,12 +8,12 @@ require 'filecluster'
 require 'utils'
 require 'daemon'
 
-$storages = []
-$tasks = {}           # tasks by storage name
-$curr_task = {}       # task by storage name
-$tasks_threads = {}   # threads by storage name
-$check_threads = {}   # threads by storage name
-$exit_signal = false 
+$storages       = []          # storages on current host
+$tasks          = {}          # tasks by storage name
+$curr_tasks     = []          # current tasks
+$tasks_threads  = {}          # threads by storage name
+$check_threads  = {}          # threads by storage name
+$exit_signal    = false 
 $global_daemon_thread = nil
 
 default_db_config = File.expand_path(File.dirname(__FILE__))+'/db.yml'

data/lib/daemon.rb

@@ -17,12 +17,12 @@ end
 
 def run_global_daemon(timeout)
   $log.debug('Run global daemon check')
-  r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
+  r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
   if !r || r['curr_time'].to_i - r['time'].to_i > timeout
     $log.debug('Set global daemon host to current')
-    FC::DB.connect.query("REPLACE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}', name='global_daemon_host'")
+    FC::DB.query("REPLACE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}', name='global_daemon_host'")
     sleep 1
-    r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
+    r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
   end
   if r['val'] == FC::Storage.curr_host
     if !$global_daemon_thread || !$global_daemon_thread.alive?
@@ -66,9 +66,10 @@ def update_tasks
     cond = "storage_name in (#{storages_names}) AND status='#{type.to_s}'"
     ids = $tasks.map{|storage_name, storage_tasks| storage_tasks.select{|task| task[:action] == type}}.
       flatten.map{|task| task[:item_storage].id}
-    $curr_task.map{|storage_name, task| ids << task[:item_storage].id if task && task[:action] == type}
+    ids += $curr_tasks.select{|task| task[:action] == type}
       
     cond << "AND id not in (#{ids.join(',')})" if (ids.length > 0)
+    cond << "LIMIT 1000"
     FC::ItemStorage.where(cond).each do |item_storage|
       $tasks[item_storage.storage_name] = [] unless $tasks[item_storage.storage_name]
       $tasks[item_storage.storage_name] << {:action => type, :item_storage => item_storage}
@@ -83,10 +84,17 @@ end
 def run_tasks
   $log.debug('Run tasks')
   $storages.each do |storage|
-    thread = $tasks_threads[storage.name]
-    if (!thread || !thread.alive?) && $tasks[storage.name] && $tasks[storage.name].size > 0
+    $tasks_threads[storage.name] = [] unless $tasks_threads[storage.name]
+    $tasks_threads[storage.name].delete_if {|thread| !thread.alive?}
+    tasks_count = $tasks[storage.name] ? $tasks[storage.name].size : 0
+    threads_count = $tasks_threads[storage.name].count
+    # 10 tasks per thread, maximum 3 tasks
+    run_threads_count = (tasks_count/10.0).ceil - threads_count
+    run_threads_count = 3 if run_threads_count > 3
+    $log.debug("tasks_count: #{tasks_count}, threads_count: #{threads_count}, run_threads_count: #{run_threads_count}")
+    run_threads_count.times do
       $log.debug("spawn TaskThread for #{storage.name}") 
-      $tasks_threads[storage.name] = TaskThread.new(storage.name)
+      $tasks_threads[storage.name] << TaskThread.new(storage.name)
     end
   end
 end

data/lib/daemon/global_daemon_thread.rb

@@ -6,9 +6,9 @@ class GlobalDaemonThread < BaseThread
       sleep timeout.to_f/2
       exit if $exit_signal
       
-      r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
+      r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
       if r['val'] == FC::Storage.curr_host
-        FC::DB.connect.query("UPDATE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}' WHERE name='global_daemon_host'")
+        FC::DB.query("UPDATE #{FC::DB.prefix}vars SET val='#{FC::Storage.curr_host}' WHERE name='global_daemon_host'")
       else
         $log.info("Exit from GlobalDaemonThread: global daemon already running on #{r['val']}")
         FC::DB.close
@@ -40,7 +40,7 @@ class GlobalDaemonThread < BaseThread
     sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name ORDER BY ist.id) as storages, p.id as policy_id, p.copies as policy_copies "+
       "FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p, #{FC::ItemStorage.table_name} as ist WHERE "+
       "i.policy_id = p.id AND ist.item_id = i.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT 1000"
-    r = FC::DB.connect.query(sql)
+    r = FC::DB.query(sql)
     r.each do |row|
       $log.info("GlobalDaemonThread: new item_storage for item #{row['item_id']}")
       item_storages = row['storages'].split(',')
@@ -61,11 +61,11 @@ class GlobalDaemonThread < BaseThread
   def delete_deleted_items
     $log.debug("GlobalDaemonThread: delete_deleted_items")
     
-    r = FC::DB.connect.query("SELECT i.id FROM #{FC::Item.table_name} as i LEFT JOIN #{FC::ItemStorage.table_name} as ist ON i.id=ist.item_id WHERE i.status = 'delete' AND ist.id IS NULL")
+    r = FC::DB.query("SELECT i.id FROM #{FC::Item.table_name} as i LEFT JOIN #{FC::ItemStorage.table_name} as ist ON i.id=ist.item_id WHERE i.status = 'delete' AND ist.id IS NULL")
     ids = r.map{|row| row['id']}
     if ids.count > 0
       ids = ids.join(',')
-      FC::DB.connect.query("DELETE FROM #{FC::Item.table_name} WHERE id in (#{ids})")
+      FC::DB.query("DELETE FROM #{FC::Item.table_name} WHERE id in (#{ids})")
       $log.info("GlobalDaemonThread: delete items #{ids}")
     end
   end

data/lib/daemon/task_thread.rb

@@ -1,38 +1,48 @@
 class TaskThread < BaseThread
   def go(storage_name)
+    return unless $tasks[storage_name]
     while task = $tasks[storage_name].shift do
-      $curr_task[storage_name] = task
+      $curr_tasks << task
       $log.debug("TaskThread(#{storage_name}): run task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
       if task[:action] == :delete
-        make_delete(task[:item_storage])
+        make_delete(task)
       elsif task[:action] == :copy
-        make_copy(task[:item_storage])
+        make_copy(task)
       else
         error "Unknown task action: #{task[:action]}"
       end
-      $curr_task[storage_name] = nil
+      $curr_tasks.delete(task)
       $log.debug("TaskThread(#{storage_name}): Finish task type=#{task[:action]}, item_storage=#{task[:item_storage].id}")
     end
   end
   
-  def make_delete(item_storage)
+  def make_delete(task)
+    item_storage = task[:item_storage]
+    # TODO: не лазить в базу за item
     storage = $storages.detect{|s| s.name == item_storage.storage_name}
     item = FC::Item.find(item_storage.item_id)
     storage.delete_file(item.name)
     item_storage.delete
   rescue Exception => e
     error "Delete item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
+    $curr_tasks.delete(task)
   end
   
-  def make_copy(item_storage)
+  def make_copy(task)
+    item_storage = task[:item_storage]
     # TODO: не лазить в базу за item, item_storages - перенести на стадию подготовки task-а
     storage = $storages.detect{|s| s.name == item_storage.storage_name}
     item = FC::Item.find(item_storage.item_id)
     src_item_storage = FC::ItemStorage.where("item_id = ? AND status = 'ready'", item.id).sample
+    unless src_item_storage
+      $log.info("Item ##{item.id} #{item.name} has no ready item_storage")
+      return nil 
+    end
     src_storage = $all_storages.detect{|s| s.name == src_item_storage.storage_name}
     $log.debug("Copy from #{src_storage.name} to #{storage.name} #{storage.path}#{item.name}")
     item.copy_item_storage(src_storage, storage, item_storage)
   rescue Exception => e
     error "Copy item_storage error: #{e.message}; #{e.backtrace.join(', ')}", :item_id => item_storage.item_id, :item_storage_id => item_storage.id
+    $curr_tasks.delete(task)
   end
 end

data/lib/fc/base.rb

@@ -34,7 +34,7 @@ module FC
     
     # get element by id
     def self.find(id)
-      r = FC::DB.connect.query("SELECT * FROM #{self.table_name} WHERE id=#{id.to_i}")
+      r = FC::DB.query("SELECT * FROM #{self.table_name} WHERE id=#{id.to_i}")
       raise "Record not found (#{self.table_name}.id=#{id})" if r.count == 0
       self.create_from_fiels(r.first)
     end
@@ -43,7 +43,7 @@ module FC
     def self.where(cond = "1", *params)
       i = -1
       sql = "SELECT * FROM #{self.table_name} WHERE #{cond.gsub('?'){i+=1; "'#{Mysql2::Client.escape(params[i].to_s)}'"}}"
-      r = FC::DB.connect.query(sql)
+      r = FC::DB.query(sql)
       r.map{|data| self.create_from_fiels(data)}
     end
     
@@ -63,7 +63,7 @@ module FC
       if fields.length > 0
         sql << fields.join(',')
         sql << " WHERE id=#{@id.to_i}" if @id
-        FC::DB.connect.query(sql)
+        FC::DB.query(sql)
         @id = FC::DB.connect.last_id unless @id
         self.class.table_fields.each do |key|
           @database_fields[key] = self.send(key)
@@ -81,7 +81,7 @@ module FC
 
     # delete object from DB
     def delete
-      FC::DB.connect.query("DELETE FROM #{self.class.table_name} WHERE id=#{@id.to_i}") if @id
+      FC::DB.query("DELETE FROM #{self.class.table_name} WHERE id=#{@id.to_i}") if @id
     end
     
   end

data/lib/fc/db.rb

@@ -35,6 +35,19 @@ module FC
       end
     end
     
+    # connect.query with deadlock solution
+    def self.query(sql)
+      FC::DB.connect.query(sql)
+    rescue Mysql2::Error => e
+      if e.message.match('Deadlock found when trying to get lock')
+        puts "Deadlock"
+        sleep 0.1
+        self.query(sql)
+      else 
+        raise e
+      end
+    end
+    
     def DB.init_db
       FC::DB.connect.query(%{
         CREATE TABLE #{@prefix}items (

data/lib/fc/item.rb

@@ -34,7 +34,7 @@ module FC
       if item
         if options[:replace] || storage
           # mark delete item_storages on replace
-          FC::DB.connect.query("UPDATE #{FC::ItemStorage.table_name} SET status='delete' WHERE item_id = #{item.id}") if options[:replace] && !storage
+          FC::DB.query("UPDATE #{FC::ItemStorage.table_name} SET status='delete' WHERE item_id = #{item.id}") if options[:replace] && !storage
           # replace all fields
           item_params.each{|key, val| item.send("#{key}=", val)}
         else
@@ -101,7 +101,7 @@ module FC
     
     # mark items_storages for delete
     def mark_deleted
-      FC::DB.connect.query("UPDATE #{FC::ItemStorage.table_name} SET status='delete' WHERE item_id = #{id}")
+      FC::DB.query("UPDATE #{FC::ItemStorage.table_name} SET status='delete' WHERE item_id = #{id}")
       self.status = 'delete'
       save
     end
@@ -115,7 +115,7 @@ module FC
     end
     
     def get_available_storages
-      r = FC::DB.connect.query("SELECT st.* FROM #{FC::Storage.table_name} as st, #{FC::ItemStorage.table_name} as ist WHERE 
+      r = FC::DB.query("SELECT st.* FROM #{FC::Storage.table_name} as st, #{FC::ItemStorage.table_name} as ist WHERE 
         ist.item_id = #{id} AND ist.status='ready' AND ist.storage_name = st.name")
       r.map{|data| FC::Storage.create_from_fiels(data)}.select {|storage| storage.up? }
     end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/lib/manage/policies.rb

@@ -11,7 +11,7 @@ end
 
 def policies_show
   if policy = find_policy
-    count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} WHERE policy_id = #{policy.id}").first['cnt']
+    count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} WHERE policy_id = #{policy.id}").first['cnt']
     puts %Q{Policy
   ID:               #{policy.id}
   Name:             #{policy.name}

data/lib/manage/show.rb

@@ -3,7 +3,7 @@ def show_current_host
 end
 
 def show_global_daemon
-  r = FC::DB.connect.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
+  r = FC::DB.query("SELECT #{FC::DB.prefix}vars.*, UNIX_TIMESTAMP() as curr_time FROM #{FC::DB.prefix}vars WHERE name='global_daemon_host'").first
   if r 
     puts "Global daemon run on #{r['val']}\nLast run #{r['curr_time']-r['time']} seconds ago."
   else
@@ -31,7 +31,7 @@ def show_host_info
   else
     puts "Info for host #{host}"
     storages.each do |storage|
-      counts = FC::DB.connect.query("SELECT status, count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}' GROUP BY status")
+      counts = FC::DB.query("SELECT status, count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}' GROUP BY status")
       str = "#{storage.name} #{size_to_human(storage.size)}/#{size_to_human(storage.size_limit)} "
       str += "#{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}"
       str += " #{storage.check_time_delay} seconds ago" if storage.check_time
@@ -46,17 +46,17 @@ end
 
 def show_items_info
   puts "Items by status:"
-  counts = FC::DB.connect.query("SELECT status, count(*) as cnt FROM #{FC::Item.table_name} WHERE 1 GROUP BY status")
+  counts = FC::DB.query("SELECT status, count(*) as cnt FROM #{FC::Item.table_name} WHERE 1 GROUP BY status")
   counts.each do |r|
     puts "   #{r['status']}: #{r['cnt']}"
   end
   puts "Items storages by status:"
-  counts = FC::DB.connect.query("SELECT status, count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE 1 GROUP BY status")
+  counts = FC::DB.query("SELECT status, count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE 1 GROUP BY status")
   counts.each do |r|
     puts "   #{r['status']}: #{r['cnt']}"
   end
-  count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.policy_id = p.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready'").first['cnt']
+  count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.policy_id = p.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready'").first['cnt']
   puts "Items to copy: #{count}"
-  count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.policy_id = p.id AND i.copies > p.copies AND i.status = 'ready'").first['cnt']
+  count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p WHERE i.policy_id = p.id AND i.copies > p.copies AND i.status = 'ready'").first['cnt']
   puts "Items to delete: #{count}"
 end

data/lib/manage/storages.rb

@@ -14,7 +14,7 @@ end
 
 def storages_show
   if storage = find_storage
-    count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
+    count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
     puts %Q{Storage
   Name:           #{storage.name}
   Host:           #{storage.host} 

data/test/base_test.rb

@@ -3,7 +3,7 @@ require 'helper'
 class BaseTest < Test::Unit::TestCase
   class << self
     def shutdown
-      FC::DB.connect.query("DELETE FROM items")
+      FC::DB.query("DELETE FROM items")
     end
   end
   

data/test/daemon_test.rb

@@ -59,10 +59,10 @@ class DaemonTest < Test::Unit::TestCase
     
     def shutdown
       Process.kill("KILL", @@pid)
-      FC::DB.connect.query("DELETE FROM items_storages")
-      FC::DB.connect.query("DELETE FROM items")
-      FC::DB.connect.query("DELETE FROM policies")
-      FC::DB.connect.query("DELETE FROM storages")
+      FC::DB.query("DELETE FROM items_storages")
+      FC::DB.query("DELETE FROM items")
+      FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM storages")
       `rm -rf /tmp/host*-sd*`
       `rm -rf #{@@test_file_path}`
       `rm -rf #{@@test_dir_path}`

data/test/db_test.rb

@@ -34,10 +34,10 @@ class DbTest < Test::Unit::TestCase
       @@item_storages_ids = item_storages.map{|is| is.save; is.id }
     end
     def shutdown
-      FC::DB.connect.query("DELETE FROM items_storages")
-      FC::DB.connect.query("DELETE FROM items")
-      FC::DB.connect.query("DELETE FROM policies")
-      FC::DB.connect.query("DELETE FROM storages")
+      FC::DB.query("DELETE FROM items_storages")
+      FC::DB.query("DELETE FROM items")
+      FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM storages")
     end
   end
   def setup

data/test/functional_test.rb

@@ -33,10 +33,10 @@ class FunctionalTest < Test::Unit::TestCase
       @@policies.each { |policy| policy.save}
     end
     def shutdown
-      FC::DB.connect.query("DELETE FROM items_storages")
-      FC::DB.connect.query("DELETE FROM items")
-      FC::DB.connect.query("DELETE FROM policies")
-      FC::DB.connect.query("DELETE FROM storages")
+      FC::DB.query("DELETE FROM items_storages")
+      FC::DB.query("DELETE FROM items")
+      FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM storages")
       `rm -rf /tmp/host*-sd*`
       `rm -rf #{@@test_file_path}`
       `rm -rf #{@@test_dir_path}`

data/test/helper.rb

@@ -10,8 +10,8 @@ TEST_USER     = 'root'
 TEST_PASSWORD = ''
 
 FC::DB.connect_by_config(:username => TEST_USER, :password => TEST_PASSWORD)
-FC::DB.connect.query("DROP DATABASE IF EXISTS #{TEST_DATABASE}")
-FC::DB.connect.query("CREATE DATABASE #{TEST_DATABASE}")
-FC::DB.connect.query("USE #{TEST_DATABASE}")
+FC::DB.query("DROP DATABASE IF EXISTS #{TEST_DATABASE}")
+FC::DB.query("CREATE DATABASE #{TEST_DATABASE}")
+FC::DB.query("USE #{TEST_DATABASE}")
 FC::DB.init_db
 FC::DB.options[:database] = TEST_DATABASE

data/test/item_test.rb

@@ -17,9 +17,9 @@ class ItemTest < Test::Unit::TestCase
       end
     end
     def shutdown
-      FC::DB.connect.query("DELETE FROM items_storages")
-      FC::DB.connect.query("DELETE FROM items")
-      FC::DB.connect.query("DELETE FROM storages")
+      FC::DB.query("DELETE FROM items_storages")
+      FC::DB.query("DELETE FROM items")
+      FC::DB.query("DELETE FROM storages")
     end
   end
   

data/test/policy_test.rb

@@ -14,8 +14,8 @@ class PolicyTest < Test::Unit::TestCase
       @@policy.save
     end
     def shutdown
-      FC::DB.connect.query("DELETE FROM policies")
-      FC::DB.connect.query("DELETE FROM storages")
+      FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM storages")
     end
   end 
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-14 00:00:00.000000000 Z
+date: 2013-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -186,7 +186,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 789694035
+      hash: -778580397
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -195,7 +195,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 789694035
+      hash: -778580397
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24