filecluster 0.0.3

data/test/daemon_test.rb

@@ -0,0 +1,98 @@
+require 'helper'
+require 'open3'
+require 'timeout'
+
+class DaemonTest < Test::Unit::TestCase
+  class << self
+    def startup
+      @debug = false #show stdout and sterr of fc-daemon
+      
+      dir = File.expand_path(File.dirname(__FILE__))
+      db_config_file = dir+'/db_test.yml'
+      daemon_bin = File.expand_path(dir+'/../bin/fc-daemon')
+      
+      File.open(db_config_file, 'w') do |f|
+        f.write(FC::DB.options.to_yaml)
+      end
+      
+      @@errors_count = FC::Error.where.count
+      
+      @stotage_checks = 0
+      Thread.new do
+        Open3.popen2e("#{daemon_bin} -c #{db_config_file} -l debug -t 1 -g 1 -h host1") do |stdin, stdout, t|
+          @@pid = t.pid
+          while line = stdout.readline
+            @stotage_checks += 1 if line =~ /Finish stotage check/i
+            puts line if @debug
+          end
+        end
+      end
+      
+      # tmp fake storages dirs
+      `rm -rf /tmp/host*-sd*`
+      `mkdir -p /tmp/host1-sda/ /tmp/host1-sdb/ /tmp/host1-sdc/`
+      
+      # test file to copy
+      @@test_file_path = '/tmp/fc_test_file'
+      `dd if=/dev/urandom of=#{@@test_file_path} bs=1M count=1 2>&1`
+      
+      @@storages = []
+      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size_limit => 1000000000)
+      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :size_limit => 1000000000)
+      @@storages << FC::Storage.new(:name => 'host1-sdc', :host => 'host1', :path => '/tmp/host1-sdc/', :size_limit => 1000000000)
+      @@storages.each { |storage| storage.save}
+      
+      @@policy = FC::Policy.new(:storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2)
+      @@policy.save
+      
+      # wait for running fc-daemon
+      Timeout::timeout(5) do
+        while @stotage_checks < @@storages.size
+          sleep 0.1
+        end
+      end
+    end
+    
+    def shutdown
+      Process.kill("KILL", @@pid)
+      FC::DB.connect.query("DELETE FROM items_storages")
+      FC::DB.connect.query("DELETE FROM items")
+      FC::DB.connect.query("DELETE FROM policies")
+      FC::DB.connect.query("DELETE FROM storages")
+      `rm -rf /tmp/host1-sda /tmp/host1-sdb /tmp/host1-sdc`
+    end
+  end
+  
+  should "daemon_all" do
+    @@storages.each {|storage| storage.reload}
+    assert @@storages[0].up?, "Storage #{@@storages[0].name} down" 
+    assert @@storages[1].up?, "Storage #{@@storages[1].name} down"
+    
+    FC::Storage.any_instance.stubs(:host).returns('host1')
+    FC::Storage.stubs(:curr_host).returns('host1')
+    assert_nothing_raised { @item1 = FC::Item.create_from_local(@@test_file_path, 'test1', @@policy, {:tag => 'test1'}) }
+    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policy, {:tag => 'test2'}) }
+    assert_nothing_raised { @item3 = FC::Item.create_from_local(@@test_file_path, 'test3', @@policy, {:tag => 'test3'}) }
+    
+    @@policy.copies = 3
+    @@policy.save
+    sleep 2
+    
+    # wait for copy
+    [1, 2, 3].each do |i|
+      ['b', 'c'].each do |j|
+        assert_equal `du -b /tmp/host1-sda/test$i 2>&1`.to_i, `du -b /tmp/host$i-sd$j/test$i 2>&1`.to_i
+      end
+    end
+        
+    @@policy.copies = 2
+    @@policy.save
+    item_storage = FC::ItemStorage.where('item_id = ? AND storage_name = ?', @item1.id, 'host1-sdc').first
+    item_storage.status = 'delete'
+    item_storage.save
+    sleep 2
+    assert_equal 0, `du -b /tmp/host1-sdc/test1 2>&1`.to_i
+    
+    assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
+  end
+end

data/test/db_test.rb

@@ -0,0 +1,182 @@
+require 'helper'
+
+class DbTest < Test::Unit::TestCase
+  class << self
+    def startup
+      storages = []
+      storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1')
+      storages << FC::Storage.new(:name => 'rec1-sdb', :host => 'rec1')
+      storages << FC::Storage.new(:name => 'rec1-sdc', :host => 'rec1')
+      storages << FC::Storage.new(:name => 'rec1-sdd', :host => 'rec1')
+      storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2')
+      storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2')
+      storages << FC::Storage.new(:name => 'rec2-sdc', :host => 'rec2')
+      storages << FC::Storage.new(:name => 'rec2-sdd', :host => 'rec2')
+      @@storages_ids = storages.map{|storage| storage.save; storage.id }
+      
+      policies = []
+      policies << FC::Policy.new(:storages => 'rec1-sda,rec1-sdd', :copies => 2)
+      policies << FC::Policy.new(:storages => 'rec1-sda,bla,rec2-sdd', :copies => 3)
+      policies << FC::Policy.new(:storages => 'bla,rec1-sda,test', :copies => 4)
+      @@policies_ids = policies.map{|policy| policy.save; policy.id }
+      
+      items = []
+      items << FC::Item.new(:name => 'test1', :policy_id => policies.first.id, :size => 150)
+      items << FC::Item.new(:name => 'test2', :policy_id => policies.first.id, :size => 200)
+      items << FC::Item.new(:name => 'test3', :policy_id => policies.first.id, :size => 400)
+      @@items_ids = items.map{|item| item.save; item.id }
+      
+      item_storages = []
+      items.each do |item|
+        item_storages << FC::ItemStorage.new(:item_id => item.id, :storage_name => 'rec1-sda')
+        item_storages << FC::ItemStorage.new(:item_id => item.id, :storage_name => 'rec2-sda')
+      end
+      @@item_storages_ids = item_storages.map{|is| is.save; is.id }
+    end
+    def shutdown
+      FC::DB.connect.query("DELETE FROM items_storages")
+      FC::DB.connect.query("DELETE FROM items")
+      FC::DB.connect.query("DELETE FROM policies")
+      FC::DB.connect.query("DELETE FROM storages")
+    end
+  end
+  def setup
+    @storages = @@storages_ids.map{|id| FC::Storage.find(id)}
+    @storage = @storages.first
+    @policies = @@policies_ids.map{|id| FC::Policy.find(id)}
+    @policy = @policies.first
+    @items = @@items_ids.map{|id| FC::Item.find(id)}
+    @item = @items.first
+    @item_storages = @@item_storages_ids.map{|id| FC::ItemStorage.find(id)}
+    @item_storage = @item_storages.first
+    @item_storage2 = @item_storages[1]
+  end
+  
+  should "items" do
+    assert @items.count > 0, 'Items not loaded'
+    @items.each{|item| assert item.time > 0, "Item (id=#{item.id}) time = 0"}
+    sleep 1
+    time = @item.time
+    @item.tag = 'blabla'
+    @item.save
+    @item.reload
+    assert_not_equal time, @item.time, "Item (id=#{@item.id}) time not changed after save"
+    item2 = FC::Item.new(:name => @item.name, :policy_id => @item.policy_id)
+    item2.save
+    assert_equal 0, item2.id, "Item (id=#{item2.id}) successfull insert on uniq key"
+  end
+  
+  should "storages" do
+    assert @storages.count > 0, 'Storages not loaded'
+    storage = FC::Storage.new(:name => 'rec1-sda', :host => 'rec1')
+    storage.save
+    assert_equal storage.id, 0, "Storage duplicate name"
+  end
+  
+  should "policies and storages" do
+    assert @policies.count > 0, 'Policies not loaded'
+    assert_equal 'rec1-sda,rec1-sdd', @policies[0].storages, "Policy (id=#{@policies[0].id}) incorrect storages"
+    assert_equal 'rec1-sda,rec2-sdd', @policies[1].storages, "Policy (id=#{@policies[0].id}) incorrect storages"
+    assert_equal 'rec1-sda', @policies[2].storages, "Policy (id=#{@policies[0].id}) incorrect storages"
+    assert_raise(Mysql2::Error, 'Create policy with incorrect storages') { FC::Policy.new(:storages => 'bla,test').save }
+    assert_raise(Mysql2::Error, 'Change storage name with linked polices') { @storages[0].name = 'blabla'; @storages[0].save }
+    assert_raise(Mysql2::Error, 'Delete storage name with linked polices') { @storages[0].delete }
+    assert_nothing_raised { @storages[6].name = 'rec2-sdc-new'; @storages[6].save }
+    @storages[3].name = 'rec1-sdd-new' #rec1-sdd
+    @storages[3].save
+    @policies[0].reload
+    assert_equal 'rec1-sda', @policies[0].storages, "Policy (id=#{@policies[0].id}) incorrect storages after storage change"
+    @storages[7].delete  #rec2-sdd
+    @@storages_ids.delete(@storages[7].id)
+    @policies[1].reload
+    assert_equal 'rec1-sda', @policies[1].storages, "Policy (id=#{@policies[1].id}) incorrect storages after storage delete"
+    @policies[0].storages = 'rec1-sda,rec2-sda,bla bla'
+    @policies[0].save
+    @policies[0].reload
+    assert_equal 'rec1-sda,rec2-sda', @policies[0].storages, "Policy (id=#{@policies[0].id}) incorrect storages after change"
+    assert_raise(Mysql2::Error, 'Save empty policy storage') { @policies[0].storages = 'blabla'; @policies[0].save }
+  end
+  
+  should "item_storages doubles" do
+    is = FC::ItemStorage.new(:item_id => @item_storage.item_id, :storage_name => @item_storage.storage_name)
+    is.save
+    assert_equal 0, is.id, 'Item_storages successfull insert on uniq key'
+  end
+  
+  should "item_storages times" do
+    assert @item_storages.count > 0, 'Item_storages not loaded'
+    @item_storages.each{|is| assert is.time > 0, "Item_storage (id=#{is.id}) time = 0"}
+    sleep 1
+    storage_name = @item_storage.storage_name
+    time = @item_storage.time
+    @item_storage.storage_name = 'rec1-sdc'
+    @item_storage.save
+    @item_storage.reload
+    assert_not_equal time, @item_storage.time, "Item_storage (id=#{@item_storage.id}) time not changed after save"
+    @item_storage.storage_name = storage_name
+    @item_storage.save
+  end
+  
+  should "item_storages references" do
+    assert_raise(Mysql2::Error, "Delete item (id=#{@items[2].id} with references to item_storages") { @items[2].delete }
+    assert_raise(Mysql2::Error, "Delete storage (id=#{@storages[4].id} with references to item_storages") { @storages[4].delete }
+  end
+  
+  should "item_storages copies, statuses, size" do
+    @items.each{|item| assert_equal 2, item.copies, "Item (id=#{item.id}) copies not inc after add item_storage"}
+    size_sum = @items.inject(0){|sum, item| sum+item.size}
+    assert_equal size_sum, @storage.size, "storage (id=#{@storage.id}) size <> ready not inc after item_storage.status='ready'"
+    
+    assert_equal 'new', @item.status, "Item (id=#{@item.id}) status <> 'new'"
+    @item_storage.status = 'ready'
+    @item_storage.save
+    @item_storage2.status = 'ready'
+    @item_storage2.save
+    @item.reload
+    assert_equal 'ready', @item.status, "Item (id=#{@item.id}) status <> 'ready' not changed after item_storage.status='ready'"
+    @item_storage.status = 'error'
+    @item_storage.save
+    @item.reload
+    assert_equal 'ready', @item.status, "Item (id=#{@item.id}) status <> 'ready' changed after one of two item_storage.status='error'"
+    @item_storage2.status = 'error'
+    @item_storage2.save
+    @item.reload
+    assert_equal 'error', @item.status, "Item (id=#{@item.id}) status <> 'error' not changed after item_storage.status='error'"
+    assert_equal 2, @item.copies, "Item (id=#{@item.id}) copies changed after item_storage.status='error'"
+    
+    @item_storage.status = 'ready'
+    @item_storage.save
+    @item.reload
+    assert_equal 'ready', @item.status, "Item (id=#{@item.id}) status <> 'ready' not changed after item_storage.status='ready'"
+    @item_storage.status = 'delete'
+    @item_storage.save
+    @item.reload
+    assert_equal 'error', @item.status, "Item (id=#{@item.id}) status <> 'error' not changed after item_storage.status='delete'"
+    assert_equal 1, @item.copies, "Item (id=#{@item.id}) copies not decreased after item_storage.status='delete'"
+    
+    @item_storage.status = 'ready'
+    @item_storage.save
+    @item_storage2.status = 'ready'
+    @item_storage2.save
+    @item.reload
+    assert_equal 'ready', @item.status, "Item (id=#{@item.id}) status <> 'ready' not changed after delete item_storage"
+    @item_storage.delete
+    @@item_storages_ids.delete(@item_storage.id)
+    @item.reload
+    @storage.reload
+    assert_equal 'ready', @item.status, "Item (id=#{@item.id}) status <> 'ready' changed after delete item_storage"
+    size_sum -= @item.size
+    assert_equal size_sum, @storage.size, "storage (id=#{@storage.id}) size not dec after delete item_storage"
+    @item_storage2.delete
+    @@item_storages_ids.delete(@item_storage2.id)
+    @item.reload
+    assert_equal 'error', @item.status, "Item (id=#{@item.id}) status <> 'error' not changed after delete all item_storage"
+  end
+  
+  should "errors time" do
+    error = FC::Error.new(:message => 'test error')
+    error.save
+    error.reload
+    assert error.time > 0, "Error (id=#{error.id}) time = 0"
+  end
+end

data/test/error_test.rb

@@ -0,0 +1,15 @@
+require 'helper'
+
+class ErrorTest < Test::Unit::TestCase
+  def setup
+    FC::Storage.stubs(:curr_host).returns('localhost')
+  end
+  
+  should "get_storages" do
+    assert_raise(RuntimeError) { FC::Error.raise 'error message', :item_id => 111 }
+    assert error = FC::Error.where('1 ORDER BY id desc LIMIT 1').first
+    assert_equal 'error message', error.message
+    assert_equal 'localhost', error.host
+    assert_equal 111, error.item_id
+  end
+end

data/test/functional_test.rb

@@ -0,0 +1,97 @@
+require 'helper'
+
+class FunctionalTest < Test::Unit::TestCase 
+    class << self
+    def startup
+      # tmp fake storages dirs
+      `rm -rf /tmp/host*-sd*`
+      `mkdir -p /tmp/host1-sda/ /tmp/host2-sda/`
+      
+      # test file to copy
+      @@test_file_path = '/tmp/fc_test_file'
+      `dd if=/dev/urandom of=#{@@test_file_path} bs=100K count=1 2>&1`
+      
+      @@storages = []
+      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size_limit => 1000000, :check_time => Time.new.to_i)
+      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :size_limit => 1000000, :check_time => Time.new.to_i)
+      @@storages << FC::Storage.new(:name => 'host2-sda', :host => 'host2', :path => '/tmp/host2-sda/', :size_limit => 1000000, :check_time => Time.new.to_i)
+      @@storages << FC::Storage.new(:name => 'host2-sdb', :host => 'host2', :path => '/tmp/host2-sdb/', :size_limit => 1000000, :check_time => Time.new.to_i)
+      @@storages << FC::Storage.new(:name => 'host3-sda', :host => 'host3', :path => '/tmp/host3-sda/', :size_limit => 1000000)
+      @@storages.each { |storage| storage.save}
+      
+      @@policies = []
+      @@policies << FC::Policy.new(:storages => 'host1-sda,host2-sda', :copies => 2)
+      @@policies << FC::Policy.new(:storages => 'host1-sdb,host2-sdb', :copies => 2)
+      @@policies << FC::Policy.new(:storages => 'host3-sda', :copies => 1)
+      @@policies << FC::Policy.new(:storages => 'host2-sda', :copies => 1)
+      @@policies.each { |policy| policy.save}
+    end
+    def shutdown
+      FC::DB.connect.query("DELETE FROM items_storages")
+      FC::DB.connect.query("DELETE FROM items")
+      FC::DB.connect.query("DELETE FROM policies")
+      FC::DB.connect.query("DELETE FROM storages")
+      `rm -rf /tmp/host1-sda /tmp/host2-sda`
+    end
+  end
+  
+  def setup
+    FC::Storage.any_instance.stubs(:host).returns('localhost')
+    FC::Storage.stubs(:curr_host).returns('localhost')
+  end
+  
+  should "item create_from_local successful" do
+    assert_nothing_raised { @item = FC::Item.create_from_local(@@test_file_path, 'test1', @@policies[0], {:tag => 'test'}) }
+    assert_kind_of FC::Item, @item
+    assert_equal `du -b /tmp/host1-sda/test1 2>&1`.to_i, `du -b #{@@test_file_path} 2>&1`.to_i
+    assert_equal `du -b /tmp/host1-sda/test1 2>&1`.to_i, @item.size
+    assert_equal 'ready', @item.status
+    item_storages = @item.get_item_storages
+    assert_equal 1, item_storages.count
+    item_storage = item_storages.first
+    assert_equal 'ready', item_storage.status
+    assert_equal 'host1-sda', item_storage.storage_name
+  end
+  
+  should "item create_from_local error local path" do
+    errors_count = FC::Error.where.count
+    assert_raise(RuntimeError) { FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[1], {:tag => 'test'}) }
+    assert_equal errors_count+1, FC::Error.where.count, "Error not saved after error local path"
+  end
+  
+  should "item create_from_local replace" do
+    @item  = FC::Item.new(:name => 'test2', :policy_id => @@policies[0].id)
+    @item.save
+    errors_count = FC::Error.where.count
+    assert_raise(RuntimeError, "replace item") { FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[0], {:tag => 'test'}) }
+    assert_equal errors_count+1, FC::Error.where.count, "Error not saved after replace item"
+    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[0], {:replace => true, :tag => 'test'}) }
+    assert_equal @item.id, @item2.id, "Item (id1=#{@item.id}, id2=#{@item2.id}) change id after replace"
+  end
+  
+  should "item create_from_local available storage" do
+    errors_count = FC::Error.where.count
+    assert_raise(RuntimeError, "available storage") { FC::Item.create_from_local(@@test_file_path, 'test3', @@policies[2], {:tag => 'test'}) }
+    assert_equal errors_count+1, FC::Error.where.count, "Error not saved on available storage"
+  end
+  
+  should "item create_from_local delete item_storage" do
+    @item  = FC::Item.new(:name => 'test4', :policy_id => @@policies[3].id)
+    @item.save
+    item_storage = FC::ItemStorage.new(:item_id => @item.id, :storage_name => 'host2-sda')
+    item_storage.save
+    assert_nothing_raised { @item = FC::Item.create_from_local(@@test_file_path, 'test4', @@policies[3], {:tag => 'test', :replace => true}) }
+    item_storages = @item.get_item_storages
+    assert_equal 1, item_storages.count
+    assert_not_equal item_storage.id, item_storages.first.id
+    assert_raise(RuntimeError) { item_storage.reload }
+  end
+  
+  should "item create_from_local check size" do
+    FC::Storage.any_instance.stubs(:copy_path => true, :file_size => 10)
+    errors_count = FC::Error.where.count
+    assert_raise(RuntimeError) { FC::Item.create_from_local(@@test_file_path, 'test5', @@policies[1], {:tag => 'test'}) }
+    assert_equal errors_count+1, FC::Error.where.count, "Error not saved after check size" 
+  end
+  
+end

data/test/helper.rb

@@ -0,0 +1,17 @@
+$:.unshift File.expand_path('../lib', File.dirname(__FILE__))
+
+require "test/unit"
+require "shoulda-context"
+require "filecluster"
+require "mocha/setup"
+
+TEST_DATABASE = 'fc_test'
+TEST_USER     = 'root'
+TEST_PASSWORD = ''
+
+FC::DB.connect_by_config(:username => TEST_USER, :password => TEST_PASSWORD)
+FC::DB.connect.query("DROP DATABASE IF EXISTS #{TEST_DATABASE}")
+FC::DB.connect.query("CREATE DATABASE #{TEST_DATABASE}")
+FC::DB.connect.query("USE #{TEST_DATABASE}")
+FC::DB.init_db
+FC::DB.options[:database] = TEST_DATABASE

data/test/item_test.rb

@@ -0,0 +1,49 @@
+require 'helper'
+
+class ItemTest < Test::Unit::TestCase
+  class << self
+    def startup
+      @@item = FC::Item.new(:name => 'test item', :policy_id => 1, :size => 150)
+      @@item.save
+      
+      @@storages = []
+      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1')
+      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2')
+      @@item_storages = @@storages.map do |storage|
+        storage.save
+        item_storage = FC::ItemStorage.new(:item_id => @@item.id, :storage_name => storage.name)
+        item_storage.save
+        item_storage
+      end
+    end
+    def shutdown
+      FC::DB.connect.query("DELETE FROM items_storages")
+      FC::DB.connect.query("DELETE FROM items")
+      FC::DB.connect.query("DELETE FROM storages")
+    end
+  end 
+
+  should "get_item_storages" do
+    assert_same_elements @@item_storages.map(&:id), @@item.get_item_storages.map(&:id)
+  end
+  
+  should "create_from_local" do
+    policy = FC::Policy.new
+    assert_raise(ArgumentError) { FC::Item.create_from_local }
+    assert_raise(ArgumentError) { FC::Item.create_from_local '/bla/bla' }
+    assert_raise(ArgumentError) { FC::Item.create_from_local '/bla/bla', 'test' }
+    assert_raise(RuntimeError) { FC::Item.create_from_local '/bla/bla', 'test', {}}
+    assert_raise(RuntimeError) { FC::Item.create_from_local '/bla/bla/bla', 'test', policy}
+  end
+  
+  should "mark_deleted" do
+    @@item.mark_deleted
+    @@item.reload
+    assert_equal 'delete', @@item.status
+    @@item_storages.each do |item_storage| 
+      item_storage.reload
+      assert_equal 'delete', item_storage.status
+    end
+  end
+  
+end