filecluster 0.5.14 → 0.5.15

data/test/daemon_test.rb

@@ -6,24 +6,24 @@ class DaemonTest < Test::Unit::TestCase
   class << self
     def startup
       @@debug = false # show stdout and sterr of fc-daemon
-      
+
       dir = File.expand_path(File.dirname(__FILE__))
       db_config_file = "#{dir}/db_test.yml"
       daemon_bin = File.expand_path("#{dir}/../bin/fc-daemon")
-      
+
       File.open(db_config_file, 'w') do |f|
         f.write(FC::DB.options.to_yaml)
       end
-      
+
       @@errors_count = FC::Error.where.count
-      
-      FC::Var.set('daemon_cycle_time', 1)
-      FC::Var.set('daemon_global_wait_time', 1)
+
+      FC::Var.set('daemon_cycle_time', 0)
+      FC::Var.set('daemon_global_wait_time', 0)
       FC::Var.set('daemon_global_error_items_storages_ttl', 2)
       FC::Var.set('daemon_global_error_items_ttl', 2)
       @stotage_checks = 0
       Thread.new do
-        Open3.popen2e("#{daemon_bin} -c #{db_config_file} -l debug -h localhost") do |stdin, stdout, t|
+        Open3.popen2e("#{daemon_bin} -c #{db_config_file} -l debug -h #{ ssh_hostname} ") do |stdin, stdout, t|
           @@pid = t.pid
           while line = stdout.readline
             @stotage_checks += 1 if line =~ /Finish stotage check/i
@@ -31,11 +31,11 @@ class DaemonTest < Test::Unit::TestCase
           end
         end
       end
-      
+
       # tmp fake storages dirs
       `rm -rf /tmp/host*-sd*`
       `mkdir -p /tmp/host1-sda/ /tmp/host1-sdb/ /tmp/host1-sdc/`
-      
+
       # test files to copy
       @@test_file_path = '/tmp/fc_test_file'
       `dd if=/dev/urandom of=#{@@test_file_path} bs=1M count=1 2>&1`
@@ -43,48 +43,48 @@ class DaemonTest < Test::Unit::TestCase
       `mkdir -p #{@@test_dir_path}/aaa #{@@test_dir_path}/bbb`
       `cp #{@@test_file_path} #{@@test_dir_path}/aaa/test1`
       `cp #{@@test_file_path} #{@@test_dir_path}/bbb/test2`
-      
+
       @@storages = []
       @@storages << FC::Storage.new(
         :name => 'host1-sda',
-        :host => 'localhost',
+        :host =>  ssh_hostname,
         :path => '/tmp/host1-sda/',
-        :copy_storages => 'host1-sdb,host1-sdc', 
+        :copy_storages => 'host1-sdb,host1-sdc',
         :size_limit => 1_000_000_000
       )
       @@storages << FC::Storage.new(
         :name => 'host1-sdb',
-        :host => 'localhost',
+        :host =>  ssh_hostname,
         :path => '/tmp/host1-sdb/',
-        :copy_storages => 'host1-sda,host1-sdc', 
+        :copy_storages => 'host1-sda,host1-sdc',
         :size_limit => 1_000_000_000
       )
       @@storages << FC::Storage.new(
         :name => 'host1-sdc',
-        :host => 'localhost',
+        :host =>  ssh_hostname,
         :path => '/tmp/host1-sdc/',
         :copy_storages => 'host1-sda,host1-sdb',
         :size_limit => 1_000_000_000
       )
       @@storages.each(&:save)
-      
+
       @@policy = FC::Policy.new(
-        :create_storages => 'host1-sda,host1-sdb,host1-sdc', 
-        :copies => 2, 
+        :create_storages => 'host1-sda,host1-sdb,host1-sdc',
+        :copies => 2,
         :name => 'policy 1',
-        :delete_deferred_time => 7
+        :delete_deferred_time => 5
       )
       @@policy.save
-      
+
       @@rule = FC::CopyRule.new(:copy_storages => 'host1-sdc', :rule => 'name == "bla/bla/test3"')
       @@rule.save
-      
+
       # wait for running fc-daemon
       Timeout.timeout(5) do
         sleep 0.1 while @stotage_checks < @@storages.size
       end
     end
-    
+
     def shutdown
       Process.kill('KILL', @@pid)
       FC::DB.query('DELETE FROM items_storages')
@@ -100,15 +100,15 @@ class DaemonTest < Test::Unit::TestCase
   should 'daemon_all' do
     puts 'Start' if @@debug
     @@storages.each(&:reload)
-    assert @@storages[0].up?, "Storage #{@@storages[0].name} down" 
+    assert @@storages[0].up?, "Storage #{@@storages[0].name} down"
     assert @@storages[1].up?, "Storage #{@@storages[1].name} down"
     assert @@storages[2].up?, "Storage #{@@storages[2].name} down"
-    
+
     assert_nothing_raised { @item1 = make_file_item('bla/bla/test1', 'test1') }
     assert_nothing_raised { @item2 = make_file_item('bla/bla/test2', 'test2') }
     assert_nothing_raised { @item3 = make_dir_item('bla/bla/test3', 'test3') }
     assert_nothing_raised { @item4 = make_file_item('bla/bla/test4', 'test4') }
-    
+
     @@policy.copies = 3
     @@policy.save
 
@@ -123,18 +123,31 @@ class DaemonTest < Test::Unit::TestCase
       end
     end
     assert_equal @@errors_count, FC::Error.where.count, 'new errors in errors table'
-    
+
     @@policy.copies = 2
     @@policy.save
+
+    item_size = `du -sb #{@@test_file_path} 2>&1`.to_i
+    storage = FC::Storage.where('name = ?', 'host1-sdc').first
+    storage.write_weight = -1
+    storage.save
     item_storage = FC::ItemStorage.where('item_id = ? AND storage_name = ?', @item1.id, 'host1-sdc').first
     item_storage.status = 'delete'
     item_storage.save
-    
+
+    sleep 2
+    puts 'Check no delete' if @@debug
+    assert_equal item_size, `du -sb /tmp/host1-sdc/bla/bla/test1 2>&1`.to_i
+
+    storage = FC::Storage.where('name = ?', 'host1-sdc').first
+    storage.write_weight = 0
+    storage.save
+
     sleep 2
     puts 'Check delete' if @@debug
     assert_equal 0, `du -sb /tmp/host1-sdc/bla/bla/test1 2>&1`.to_i
     assert_equal @@errors_count, FC::Error.where.count, 'new errors in errors table'
-    
+
     @item1.immediate_delete
     @item4.mark_deleted
     @item2.get_item_storages.each do |ist|
@@ -143,8 +156,8 @@ class DaemonTest < Test::Unit::TestCase
     end
     @item3.status = 'error'
     @item3.save
-    
-    sleep 6
+
+    sleep 4
     puts 'Check immediate_delete' if @@debug
     assert_raise(RuntimeError, 'Item not deleted after mark_deleted') { @item1.reload }
     assert_equal 0, @item2.get_item_storages.count, "ItemStorages not deleted after status='error'"
@@ -154,7 +167,7 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal 'deferred_delete', @item4.status, 'Item not deferred_delete after mark_deleted'
     assert_same_elements %w(ready ready ready), @item4.get_item_storages.map(&:status)
 
-    sleep 6
+    sleep 4
     puts 'Check mark_deleted' if @@debug
     assert_raise(RuntimeError, 'Item not deleted after mark_deleted') { @item4.reload }
   end

data/test/db_test.rb

@@ -13,19 +13,19 @@ class DbTest < Test::Unit::TestCase
       storages << FC::Storage.new(:name => 'rec2-sdc', :host => 'rec2')
       storages << FC::Storage.new(:name => 'rec2-sdd', :host => 'rec2')
       @@storages_ids = storages.map{|storage| storage.save; storage.id }
-      
+
       policies = []
       policies << FC::Policy.new(:create_storages => 'rec1-sda,rec1-sdd', :copies => 2, :name => 'policy 1')
       policies << FC::Policy.new(:create_storages => 'rec1-sda,bla,rec2-sdd', :copies => 3, :name => 'policy 2')
       policies << FC::Policy.new(:create_storages => 'bla,rec1-sda,test', :copies => 4, :name => 'policy 3')
       @@policies_ids = policies.map{|policy| policy.save; policy.id }
-      
+
       items = []
       items << FC::Item.new(:name => 'test1', :policy_id => policies.first.id, :size => 150)
       items << FC::Item.new(:name => 'test2', :policy_id => policies.first.id, :size => 200)
       items << FC::Item.new(:name => 'test3', :policy_id => policies.first.id, :size => 400)
       @@items_ids = items.map{|item| item.save; item.id }
-      
+
       item_storages = []
       items.each do |item|
         item_storages << FC::ItemStorage.new(:item_id => item.id, :storage_name => 'rec1-sda')
@@ -54,7 +54,7 @@ class DbTest < Test::Unit::TestCase
 
   should 'sql logger' do
     FC::DB.logger = mock
-    FC::DB.logger.expects(:debug).at_least_once 
+    FC::DB.logger.expects(:debug).at_least_once
     FC::DB.query 'SELECT 1'
     FC::DB.logger = nil
   end
@@ -64,7 +64,7 @@ class DbTest < Test::Unit::TestCase
     File.open(db_config_file, 'w') do |f|
       f.write(FC::DB.options.to_yaml)
     end
-    
+
     connect = FC::DB.connect
     FC::DB.close
     assert_false connect.ping, 'Mysql2 connect not closed after close call'
@@ -88,7 +88,7 @@ class DbTest < Test::Unit::TestCase
     FC::DB.connect!(:multi_threads => true)
     assert_true FC::DB.connect.ping, 'Not connected after lazy_connect with Mysql2'
     assert_true FC::DB.options[:multi_threads], 'Options from connect!(options) was not setted'
-    
+
     FC::DB.close
     FC::DB.lazy_connect do
       FC::DB.connect_by_config(FC::DB.options)
@@ -111,13 +111,20 @@ class DbTest < Test::Unit::TestCase
   should 'mysql errors' do
     FC::DB.connect!(:reconnect => true)
     FC::DB.logger = mock
-    FC::DB.logger.expects(:info).at_least_once 
+    FC::DB.logger.expects(:info).at_least_once
     assert_raise(Mysql2::Error) { FC::DB.query('retertert') }
     assert_raise(RuntimeError) { FC::DB.query('select ewrwerwerwer()') }
     FC::DB.connect!(:reconnect => false)
     FC::DB.logger = nil
   end
-  
+
+  should "item time update" do
+    item = FC::Item.new(:name => 'bla bla bla', :policy_id => @item.policy_id)
+    item.save
+    item.reload
+    assert((item.time - Time.new.to_i).abs <= 1)
+  end
+
   should "items" do
     assert @items.count > 0, 'Items not loaded'
     @items.each{|item| assert item.time > 0, "Item (id=#{item.id}) time = 0"}
@@ -131,25 +138,25 @@ class DbTest < Test::Unit::TestCase
     item2.save
     assert_equal 0, item2.id, "Item (id=#{item2.id}) successfull insert on uniq key"
   end
-  
+
   should "storages" do
     assert @storages.count > 0, 'Storages not loaded'
     storage = FC::Storage.new(:name => 'rec1-sda', :host => 'rec1')
     storage.save
     assert_equal storage.id, 0, "Storage duplicate name"
   end
-  
+
   should "policies and storages" do
     assert @policies.count > 0, 'Policies not loaded'
     assert_equal 'rec1-sda,rec1-sdd', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
     assert_equal 'rec1-sda,rec2-sdd', @policies[1].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
     assert_equal 'rec1-sda', @policies[2].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
-    
+
     FC::Policy.new(:create_storages => 'rec2-sda,rec2-sdd', :name => 'policy 1').save
     assert_equal 'rec1-sda,rec1-sdd', FC::Policy.where('name = ?', 'policy 1').first.create_storages, "Create policy with uniq name"
-    FC::Policy.new(:create_storages => 'bla,test', :name => 'new policy').save
+    FC::Policy.new(:create_storages => 'bla,test', :name => 'new policy').save  rescue nil # nil from mysql 5.6
     assert_nil FC::Policy.where('name = ?', 'new policy').first, "Create policy with incorrect create_storages"
-    
+
     assert_raise(Mysql2::Error, 'Change storage name with linked polices') { @storages[0].name = 'blabla'; @storages[0].save }
     assert_raise(Mysql2::Error, 'Delete storage name with linked polices') { @storages[0].delete }
     assert_nothing_raised { @storages[6].name = 'rec2-sdc-new'; @storages[6].save }
@@ -169,13 +176,13 @@ class DbTest < Test::Unit::TestCase
     assert_equal 'rec2-sda,rec1-sda', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages after change"
     assert_raise(Mysql2::Error, 'Save empty policy storage') { @policies[0].create_storages = 'blabla'; @policies[0].save }
   end
-  
+
   should "item_storages doubles" do
     is = FC::ItemStorage.new(:item_id => @item_storage.item_id, :storage_name => @item_storage.storage_name)
     is.save
     assert_equal 0, is.id, 'Item_storages successfull insert on uniq key'
   end
-  
+
   should "item_storages times" do
     assert @item_storages.count > 0, 'Item_storages not loaded'
     @item_storages.each{|is| assert is.time > 0, "Item_storage (id=#{is.id}) time = 0"}
@@ -189,17 +196,17 @@ class DbTest < Test::Unit::TestCase
     @item_storage.storage_name = storage_name
     @item_storage.save
   end
-  
+
   should "item_storages references" do
     assert_raise(Mysql2::Error, "Delete item (id=#{@items[2].id} with references to item_storages") { @items[2].delete }
     assert_raise(Mysql2::Error, "Delete storage (id=#{@storages[4].id} with references to item_storages") { @storages[4].delete }
   end
-  
+
   should "item_storages copies, statuses, size" do
     @items.each{|item| assert_equal 2, item.copies, "Item (id=#{item.id}) copies not inc after add item_storage"}
     size_sum = @items.inject(0){|sum, item| sum+item.size}
     assert_equal size_sum, @storage.size, "storage (id=#{@storage.id}) size <> ready not inc after item_storage.status='ready'"
-    
+
     assert_equal 'new', @item.status, "Item (id=#{@item.id}) status <> 'new'"
     @item_storage.status = 'ready'
     @item_storage.save
@@ -216,7 +223,7 @@ class DbTest < Test::Unit::TestCase
     @item.reload
     assert_equal 'error', @item.status, "Item (id=#{@item.id}) status <> 'error' not changed after item_storage.status='error'"
     assert_equal 2, @item.copies, "Item (id=#{@item.id}) copies changed after item_storage.status='error'"
-    
+
     @item_storage.status = 'ready'
     @item_storage.save
     @item.reload
@@ -226,7 +233,7 @@ class DbTest < Test::Unit::TestCase
     @item.reload
     assert_equal 'error', @item.status, "Item (id=#{@item.id}) status <> 'error' not changed after item_storage.status='delete'"
     assert_equal 1, @item.copies, "Item (id=#{@item.id}) copies not decreased after item_storage.status='delete'"
-    
+
     @item_storage.status = 'ready'
     @item_storage.save
     @item_storage2.status = 'ready'
@@ -245,7 +252,7 @@ class DbTest < Test::Unit::TestCase
     @item.reload
     assert_equal 'error', @item.status, "Item (id=#{@item.id}) status <> 'error' not changed after delete all item_storage"
   end
-  
+
   should "errors time" do
     error = FC::Error.new(:message => 'test error')
     error.save

data/test/error_test.rb

@@ -2,14 +2,14 @@ require 'helper'
 
 class ErrorTest < Test::Unit::TestCase
   def setup
-    FC::Storage.stubs(:curr_host).returns('localhost')
+    FC::Storage.stubs(:curr_host).returns(ssh_hostname)
   end
-  
+
   should "get_storages" do
     assert_raise(RuntimeError) { FC::Error.raise 'error message', :item_id => 111 }
     assert error = FC::Error.where('1 ORDER BY id desc LIMIT 1').first
     assert_equal 'error message', error.message
-    assert_equal 'localhost', error.host
+    assert_equal ssh_hostname, error.host
     assert_equal 111, error.item_id
   end
 end

data/test/functional_test.rb

@@ -1,12 +1,12 @@
 require 'helper'
 
-class FunctionalTest < Test::Unit::TestCase 
+class FunctionalTest < Test::Unit::TestCase
   class << self
     def startup
       # tmp fake storages dirs
       `rm -rf /tmp/host*-sd*`
       `mkdir -p /tmp/host1-sda/ /tmp/host2-sda/`
-      
+
       # test file to copy
       @@test_file_path = '/tmp/fc test_file "~!@#$%^&*()_+|\';'
       `dd if=/dev/urandom of=#{@@test_file_path.shellescape} bs=100K count=1 2>&1`
@@ -14,7 +14,7 @@ class FunctionalTest < Test::Unit::TestCase
       `mkdir -p #{@@test_dir_path.shellescape}/aaa #{@@test_dir_path.shellescape}/bbb`
       `cp #{@@test_file_path.shellescape} #{@@test_dir_path.shellescape}/aaa/test1`
       `cp #{@@test_file_path.shellescape} #{@@test_dir_path.shellescape}/bbb/test2`
-      
+
       @@storages = []
       @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size => 0, :size_limit => 1000000, :check_time => Time.new.to_i)
       @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :size => 0, :size_limit => 1000000, :check_time => Time.new.to_i)
@@ -22,7 +22,7 @@ class FunctionalTest < Test::Unit::TestCase
       @@storages << FC::Storage.new(:name => 'host2-sdb', :host => 'host2', :path => '/tmp/host2-sdb/', :size => 10, :size_limit => 1000000, :check_time => Time.new.to_i)
       @@storages << FC::Storage.new(:name => 'host3-sda', :host => 'host3', :path => '/tmp/host3-sda/', :size => 100, :size_limit => 1000000)
       @@storages.each { |storage| storage.save}
-      
+
       @@policies = []
       @@policies << FC::Policy.new(:create_storages => 'host1-sda,host2-sda', :copy_storages => 'host1-sdb', :copies => 2, :name => 'policy 1')
       @@policies << FC::Policy.new(:create_storages => 'host1-sdb,host2-sdb', :copy_storages => 'host1-sdb', :copies => 2, :name => 'policy 2')
@@ -40,10 +40,10 @@ class FunctionalTest < Test::Unit::TestCase
       #`rm -rf #{@@test_dir_path.shellescape}`
     end
   end
-  
+
   def setup
-    FC::Storage.any_instance.stubs(:host).returns('localhost')
-    FC::Storage.stubs(:curr_host).returns('localhost')
+    FC::Storage.any_instance.stubs(:host).returns(ENV['SSH_HOST'] || 'localhost')
+    FC::Storage.stubs(:curr_host).returns(ENV['SSH_HOST'] || 'localhost')
   end
 
   def stub_method(obj, method, method_impl)
@@ -65,7 +65,7 @@ class FunctionalTest < Test::Unit::TestCase
     assert_equal 'ready', item_storage.status
     assert_equal 'host1-sda', item_storage.storage_name
   end
-  
+
   should "item create_from_local dir successful" do
     assert_nothing_raised { @item = FC::Item.create_from_local(@@test_dir_path, '/bla/bla/test_dir', @@policies[0], {:tag => 'test_dir'}) }
     assert_kind_of FC::Item, @item
@@ -79,7 +79,7 @@ class FunctionalTest < Test::Unit::TestCase
     assert_equal 'ready', item_storage.status
     assert_equal 'host1-sda', item_storage.storage_name
   end
-  
+
   should "item create_from_local replace" do
     @item = FC::Item.new(:name => 'test2', :policy_id => @@policies[3].id)
     @item.save
@@ -97,13 +97,13 @@ class FunctionalTest < Test::Unit::TestCase
     assert_equal 'delete', item_storages['host1-sdb'].status
     assert_equal 'ready', item_storages['host2-sda'].status
   end
-  
+
   should "item create_from_local available storage" do
     errors_count = FC::Error.where.count
     assert_raise(RuntimeError, "available storage") { FC::Item.create_from_local(@@test_file_path, 'test3', @@policies[2], {:tag => 'test'}) }
     assert_equal errors_count+1, FC::Error.where.count, "Error not saved on available storage"
   end
-  
+
   should "item create_from_local delete item_storage" do
     @item  = FC::Item.new(:name => 'test4', :policy_id => @@policies[3].id)
     @item.save
@@ -115,7 +115,7 @@ class FunctionalTest < Test::Unit::TestCase
     assert_not_equal item_storage.id, item_storages.first.id
     assert_raise(RuntimeError) { item_storage.reload }
   end
-  
+
   should "item create_from_local check md5" do
     errors_count = FC::Error.where.count
     @item = FC::Item.create_from_local(@@test_file_path, 'test5', @@policies[0], {:tag => 'test'})
@@ -197,14 +197,14 @@ class FunctionalTest < Test::Unit::TestCase
     `cp #{@@test_file_path.shellescape} #{tmp_file_path.shellescape}`
     assert_nothing_raised { FC::Item.create_from_local(tmp_file_path, 'inplace test', @@policies[0]) }
   end
-  
+
   should "item create_from_local inplace for dir" do
     tmp_dir_path = "/tmp/host2-sda/inplace test dir/"
     `mkdir #{tmp_dir_path.shellescape}`
     `cp #{@@test_file_path.shellescape} #{tmp_dir_path.shellescape}`
     assert_nothing_raised { FC::Item.create_from_local(tmp_dir_path, '/inplace test dir/', @@policies[0]) }
   end
-  
+
   should "item create_from_local with move and delete" do
     tmp_file_path = "/tmp/fc test file for delete"
     `cp #{@@test_file_path.shellescape} #{tmp_file_path.shellescape}`
@@ -218,22 +218,23 @@ class FunctionalTest < Test::Unit::TestCase
   end
 
   should 'item create_from_local with block for choose storage' do
-    item = FC::Item.create_from_local(@@test_file_path, 
-                                      '/bla/bla/test7', 
-                                      @@policies[0], 
+    item = FC::Item.create_from_local(@@test_file_path,
+                                      '/bla/bla/test7',
+                                      @@policies[0],
                                       :tag => 'test') do |storages|
       storages.select { |s| s.name == 'host2-sda' }
     end
     assert_kind_of FC::Item, item
+
     assert_equal `du -sb /tmp/host2-sda/bla/bla/test7 2>&1`.to_i, item.size
     assert_equal 'ready', item.status
     item_storages = item.get_item_storages
     assert_equal 1, item_storages.count
     assert_equal 'ready', item_storages.first.status
     assert_equal 'host2-sda', item_storages.first.storage_name
-    item = FC::Item.create_from_local(@@test_file_path, 
-                                      '/bla/bla/test8', 
-                                      @@policies[0], 
+    item = FC::Item.create_from_local(@@test_file_path,
+                                      '/bla/bla/test8',
+                                      @@policies[0],
                                       :tag => 'test') { [@@storages[4]] }
     assert_equal 'host3-sda', item.get_item_storages.first.storage_name
   end