filecluster 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 016165196d545cc809e342d8d7a573087f4f5d31
-  data.tar.gz: 90171541e2b6c1f81581bd81c9475bf556d9c859
+  metadata.gz: 28654ba4883c2627f9e15f503baadaca8f3ab38d
+  data.tar.gz: d7029275d6e2ff9977c1698651786174b1c9921d
 SHA512:
-  metadata.gz: 974f810d8c7bc2f37851786f3549021e0d100d887b5c2dae805fd2e7134358b337f579ac054cae6c5a305e57e1a39d6fce54ff3136291bf9f06790ff42e72dd6
-  data.tar.gz: 792e72c575b4a91655283a2a0dc11c760949bc896c31fb0f8cd5d87dabff76059b579b4c9085695d46180b0d5330f5eb30f89f7a58fca98d26648b82948e5587
+  metadata.gz: f6b4ea85f5adb7a80580ac90227c10c4bb60df3f5adce2cb5f76d9b1aad95e5c5562bc258e51b655764afdfc8f1f5b1a702575559aaa10864d17a93a10056e56
+  data.tar.gz: 7d6dd115fab9ab6d512d389329291b59d841281e8729314b08e685cda5530b6d2ad0f8b7bbdf113710dce5bf31246581f80fd2ea0a62a365e658c055d30fa2c5

data/lib/daemon/global_daemon_thread.rb

@@ -15,6 +15,7 @@ class GlobalDaemonThread < BaseThread
       end
       
       make_item_copies
+      make_item_mark_deleted
       make_deleted_error_items_storages
       make_deleted_error_items
       delete_deleted_items
@@ -68,6 +69,28 @@ class GlobalDaemonThread < BaseThread
       end
     end
   end
+
+  # mark deleted for deferred_delete items
+  def make_item_mark_deleted
+    $log.debug("GlobalDaemonThread: make_item_mark_deleted")
+    
+    limit = FC::Var.get('daemon_global_tasks_group_limit', 1000).to_i
+    loop do
+      r = FC::DB.query %(
+        SELECT i.id FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p 
+        WHERE 
+          i.status = 'deferred_delete' AND i.policy_id = p.id AND 
+          i.time + p.delete_deferred_time < UNIX_TIMESTAMP()
+        LIMIT #{limit}
+      )
+      break if r.count == 0
+      ids = r.map{|e| e['id']}.join(',')
+      $log.info("GlobalDaemonThread: mark delete items: #{ids}")
+      FC::DB.query("UPDATE #{FC::ItemStorage.table_name} SET status='delete' WHERE item_id in (#{ids})")
+      FC::DB.query("UPDATE #{FC::Item.table_name} SET status='delete' WHERE id in (#{ids})")
+      sleep 1
+    end
+  end
   
   def delete_deleted_items
     $log.debug("GlobalDaemonThread: delete_deleted_items")

data/lib/fc/db.rb

@@ -10,18 +10,22 @@ module FC
     def self.options_yml_path
       File.expand_path(File.dirname(__FILE__) + '../../../bin/db.yml')
     end
+
+    def self.symbolize_keys(options)
+      options.each_with_object({}) { |el, memo| memo[el[0].to_sym] = el[1] }
+    end
     
     def self.connect_by_config(options)
-      @options = options.clone
-      @options[:port] = @options[:port].to_i if @options[:port]
+      @options = symbolize_keys(options)
       @prefix = @options[:prefix].to_s if @options[:prefix]
       @connects = {} unless @connects
+      @connect_block = nil
       @connects[Thread.current.object_id] = Mysql2::Client.new(@options)
     end
 
     def self.connect_by_yml(options = {})
-      db_options = Psych.load(File.read(options_yml_path))
-      connect_by_config(db_options.merge(options))
+      db_options = symbolize_keys(Psych.load(File.read(options_yml_path)))
+      connect_by_config(db_options.merge(symbolize_keys(options)))
     end
 
     def self.connect_by_active_record(options = {})
@@ -30,9 +34,10 @@ module FC
       else
         connection = ActiveRecord::Base.connection.instance_variable_get(:@connection)
       end
-      @options = connection.query_options.clone
-      @options.merge!(options)
+      @options = symbolize_keys(connection.query_options)
+      @options.merge!(symbolize_keys(options))
       @prefix = @options[:prefix].to_s if @options[:prefix]
+      @connect_block = nil
       @connects = {} unless @connects
       @connects[Thread.current.object_id] = connection
     end
@@ -43,7 +48,7 @@ module FC
 
     def self.connect_by_block(options = {})
       connection = @connect_block.call
-      @options = connection.query_options.clone.merge(options)
+      @options = connection.query_options.clone.merge(symbolize_keys(options))
       @prefix = @options[:prefix].to_s if @options[:prefix]
       @connects = {} unless @connects
       @connects[Thread.current.object_id] = connection
@@ -68,7 +73,7 @@ module FC
       elsif options[:host] || options[:database] || options[:username] || options[:password]
         connect_by_config(options)
       elsif @options
-        connect_by_config(@options.merge(options))
+        connect_by_config(@options.merge(symbolize_keys(options)))
       elsif !@no_active_record && defined?(ActiveRecord::Base) && ActiveRecord::Base.connection
         connect_by_active_record(options)
       else
@@ -318,5 +323,10 @@ module FC
     def self.migrate_2
       FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN dc varchar(255) DEFAULT ''")
     end
+
+    def self.migrate_3
+      FC::DB.query("ALTER TABLE #{@prefix}items MODIFY COLUMN status ENUM('new', 'ready', 'error', 'delete', 'deferred_delete') NOT NULL DEFAULT 'new'")
+      FC::DB.query("ALTER TABLE #{@prefix}policies ADD COLUMN delete_deferred_time int NOT NULL DEFAULT 0")
+    end
   end
 end

data/lib/fc/item.rb

@@ -13,7 +13,7 @@ module FC
     #   :additional_fields - hash of additional FC:Item fields 
     #   :no_md5 - don't use md5
     # If item_name is part of local_path it processed as inplace - local_path is valid path to the item for policy
-    def self.create_from_local(local_path, item_name, policy, options={})
+    def self.create_from_local(local_path, item_name, policy, options={}, &block)
       raise 'Path not exists' unless File.exists?(local_path)
       raise 'Policy is not FC::Policy' unless policy.instance_of?(FC::Policy)
       item_params = options.merge({
@@ -65,7 +65,12 @@ module FC
             storages.select{|s| storages_names.detect(s.name)}
           end
         end
-        storage = policy.get_proper_storage_for_create(item.size, local_path) unless storage
+        if block_given?
+          storage ||= FC::Storage.select_proper_storage_for_create(policy.get_create_storages, 
+                                                                   item.size, &block)
+        else 
+          storage ||= policy.get_proper_storage_for_create(item.size, local_path)
+        end
         FC::Error.raise 'No available storage', :item_id => item.id unless storage
         
         # mark delete item_storages on replace
@@ -130,8 +135,15 @@ module FC
       end
     end
     
-    # mark items_storages for delete
+    # mark item and his items_storages for deferred delete
+    # real delete after policy.delete_deferred_time
     def mark_deleted
+      self.status = 'deferred_delete'
+      save
+    end
+
+    # mark item and his items_storages for immediate delete
+    def immediate_delete
       FC::DB.query("UPDATE #{FC::ItemStorage.table_name} SET status='delete' WHERE item_id = #{id}")
       self.status = 'delete'
       save

data/lib/fc/policy.rb

@@ -2,7 +2,7 @@
 
 module FC
   class Policy < DbBase
-    set_table :policies, 'name, create_storages, copies'
+    set_table :policies, 'name, create_storages, copies, delete_deferred_time'
     
     class << self
       attr_accessor :storages_cache_time, :get_create_storages_mutex

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = '0.5.2'.freeze
+  VERSION = '0.5.3'.freeze
 end

data/lib/manage/item.rb

@@ -91,13 +91,16 @@ def item_rm
   if !item
     puts "Item #{name} not found."
   else
-    s = Readline.readline("Delete? (y/n) ", false).strip.downcase
-    puts ""
-    if s == "y" || s == "yes"
-      item.mark_deleted
-      puts "ok"
+    s = Readline.readline('Immediate delete? (y/n) ', false).strip.downcase
+    puts ''
+    immediate_delete = s == 'y' || s == 'yes'
+    s = Readline.readline('Delete? (y/n) ', false).strip.downcase
+    puts ''
+    if s == 'y' || s == 'yes'
+      immediate_delete ? item.immediate_delete : item.mark_deleted
+      puts 'ok'
     else
-      puts "Canceled."
+      puts 'Canceled.'
     end
   end
 end

data/lib/manage/policies.rb

@@ -13,11 +13,12 @@ def policies_show
   if policy = find_policy
     count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} WHERE policy_id = #{policy.id}").first['cnt']
     puts %Q{Policy
-  ID:               #{policy.id}
-  Name:             #{policy.name}
-  Create storages:  #{policy.create_storages}
-  Copies:           #{policy.copies}
-  Items:            #{count}}
+  ID:                   #{policy.id}
+  Name:                 #{policy.name}
+  Create storages:      #{policy.create_storages}
+  Copies:               #{policy.copies}
+  Delete deferred time: #{policy.delete_deferred_time}
+  Items:                #{count}}
   end
 end
 
@@ -26,20 +27,27 @@ def policies_add
   name = stdin_read_val('Name')
   create_storages = stdin_read_val('Create storages')
   copies = stdin_read_val('Copies').to_i
+  delete_deferred_time = stdin_read_val('Delete deferred time').to_i
   
   storages = FC::Storage.where.map(&:name)
   create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
   
   begin
-    policy = FC::Policy.new(:name => name, :create_storages => create_storages, :copies => copies)
+    policy = FC::Policy.new(
+      :name => name,
+      :create_storages => create_storages,
+      :copies => copies,
+      :delete_deferred_time => delete_deferred_time
+    )
   rescue Exception => e
     puts "Error: #{e.message}"
     exit
   end
   puts %Q{\nPolicy
-  Name:             #{name}
-  Create storages:  #{create_storages}
-  Copies:           #{copies}}
+  Name:                 #{name}
+  Create storages:      #{create_storages}
+  Copies:               #{copies}
+  Delete deferred time: #{delete_deferred_time}}
   s = Readline.readline("Continue? (y/n) ", false).strip.downcase
   puts ""
   if s == "y" || s == "yes"
@@ -74,6 +82,7 @@ def policies_change
     name = stdin_read_val("Name (now #{policy.name})", true)
     create_storages = stdin_read_val("Create storages (now #{policy.create_storages})", true)
     copies = stdin_read_val("Copies (now #{policy.copies})", true)
+    delete_deferred_time = stdin_read_val("Delete deferred time (now #{policy.delete_deferred_time})", true)
     
     storages = FC::Storage.where.map(&:name)
     create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless create_storages.empty?
@@ -83,9 +92,10 @@ def policies_change
     policy.copies = copies.to_i unless copies.empty?
     
     puts %Q{\nStorage
-    Name:             #{policy.name}
-    Create storages:  #{policy.create_storages}
-    Copies:           #{policy.copies}}
+    Name:                 #{policy.name}
+    Create storages:      #{policy.create_storages}
+    Copies:               #{policy.copies}
+    Delete deferred time: #{policy.delete_deferred_time}}
     s = Readline.readline("Continue? (y/n) ", false).strip.downcase
     puts ""
     if s == "y" || s == "yes"
@@ -109,4 +119,4 @@ def find_policy
   policy = FC::Policy.where('name = ?', ARGV[2]).first unless policy
   puts "Policy #{ARGV[2]} not found." unless policy
   policy
-end
+end

data/lib/manage/storages.rb

@@ -237,7 +237,7 @@ def make_storages_sync(storage, make_delete, silent = false, no_reconnect = fals
         db_items[path][0] = true
         next if db_items[path][1] && db_items[path][2] != 'delete'
       end
-      delete_files << path if File.file?(f)
+      delete_files << path if File.file?(f) && path != 'healthcheck'
       process_storage_dir_sync.call(path+'/') if File.directory?(f)
     end
   end  

data/test/daemon_test.rb

@@ -5,11 +5,11 @@ require 'timeout'
 class DaemonTest < Test::Unit::TestCase
   class << self
     def startup
-      @debug = false #show stdout and sterr of fc-daemon
+      @@debug = false # show stdout and sterr of fc-daemon
       
       dir = File.expand_path(File.dirname(__FILE__))
-      db_config_file = dir+'/db_test.yml'
-      daemon_bin = File.expand_path(dir+'/../bin/fc-daemon')
+      db_config_file = "#{dir}/db_test.yml"
+      daemon_bin = File.expand_path("#{dir}/../bin/fc-daemon")
       
       File.open(db_config_file, 'w') do |f|
         f.write(FC::DB.options.to_yaml)
@@ -27,7 +27,7 @@ class DaemonTest < Test::Unit::TestCase
           @@pid = t.pid
           while line = stdout.readline
             @stotage_checks += 1 if line =~ /Finish stotage check/i
-            puts line if @debug
+            puts line if @@debug
           end
         end
       end
@@ -45,62 +45,86 @@ class DaemonTest < Test::Unit::TestCase
       `cp #{@@test_file_path} #{@@test_dir_path}/bbb/test2`
       
       @@storages = []
-      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_storages => 'host1-sdb,host1-sdc', :size_limit => 1000000000)
-      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :copy_storages => 'host1-sda,host1-sdc', :size_limit => 1000000000)
-      @@storages << FC::Storage.new(:name => 'host1-sdc', :host => 'host1', :path => '/tmp/host1-sdc/', :copy_storages => 'host1-sda,host1-sdb', :size_limit => 1000000000)
-      @@storages.each { |storage| storage.save}
+      @@storages << FC::Storage.new(
+        :name => 'host1-sda',
+        :host => 'host1',
+        :path => '/tmp/host1-sda/',
+        :copy_storages => 'host1-sdb,host1-sdc', 
+        :size_limit => 1_000_000_000
+      )
+      @@storages << FC::Storage.new(
+        :name => 'host1-sdb',
+        :host => 'host1',
+        :path => '/tmp/host1-sdb/',
+        :copy_storages => 'host1-sda,host1-sdc', 
+        :size_limit => 1_000_000_000
+      )
+      @@storages << FC::Storage.new(
+        :name => 'host1-sdc',
+        :host => 'host1',
+        :path => '/tmp/host1-sdc/',
+        :copy_storages => 'host1-sda,host1-sdb',
+        :size_limit => 1_000_000_000
+      )
+      @@storages.each(&:save)
       
-      @@policy = FC::Policy.new(:create_storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2, :name => 'policy 1')
+      @@policy = FC::Policy.new(
+        :create_storages => 'host1-sda,host1-sdb,host1-sdc', 
+        :copies => 2, 
+        :name => 'policy 1',
+        :delete_deferred_time => 7
+      )
       @@policy.save
       
       @@rule = FC::CopyRule.new(:copy_storages => 'host1-sdc', :rule => 'name == "bla/bla/test3"')
       @@rule.save
       
       # wait for running fc-daemon
-      Timeout::timeout(5) do
-        while @stotage_checks < @@storages.size
-          sleep 0.1
-        end
+      Timeout.timeout(5) do
+        sleep 0.1 while @stotage_checks < @@storages.size
       end
     end
     
     def shutdown
-      Process.kill("KILL", @@pid)
-      FC::DB.query("DELETE FROM items_storages")
-      FC::DB.query("DELETE FROM items")
-      FC::DB.query("DELETE FROM policies")
-      FC::DB.query("DELETE FROM storages")
+      Process.kill('KILL', @@pid)
+      FC::DB.query('DELETE FROM items_storages')
+      FC::DB.query('DELETE FROM items')
+      FC::DB.query('DELETE FROM policies')
+      FC::DB.query('DELETE FROM storages')
       `rm -rf /tmp/host*-sd*`
       `rm -rf #{@@test_file_path}`
       `rm -rf #{@@test_dir_path}`
     end
   end
 
-  should "daemon_all" do
-    puts 'Start' if @debug
-    @@storages.each {|storage| storage.reload}
+  should 'daemon_all' do
+    puts 'Start' if @@debug
+    @@storages.each(&:reload)
     assert @@storages[0].up?, "Storage #{@@storages[0].name} down" 
     assert @@storages[1].up?, "Storage #{@@storages[1].name} down"
     assert @@storages[2].up?, "Storage #{@@storages[2].name} down"
     
     FC::Storage.any_instance.stubs(:host).returns('host1')
     FC::Storage.stubs(:curr_host).returns('host1')
-    assert_nothing_raised { @item1 = FC::Item.create_from_local(@@test_file_path, 'bla/bla/test1', @@policy, {:tag => 'test1'}) }
-    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'bla/bla/test2', @@policy, {:tag => 'test2'}) }
-    assert_nothing_raised { @item3 = FC::Item.create_from_local(@@test_dir_path, 'bla/bla/test3', @@policy, {:tag => 'test3'}) }
+    assert_nothing_raised { @item1 = make_file_item('bla/bla/test1', 'test1') }
+    assert_nothing_raised { @item2 = make_file_item('bla/bla/test2', 'test2') }
+    assert_nothing_raised { @item3 = make_dir_item('bla/bla/test3', 'test3') }
+    assert_nothing_raised { @item4 = make_file_item('bla/bla/test4', 'test4') }
     
     @@policy.copies = 3
     @@policy.save
 
     # wait for copy
     sleep 2
-    puts 'Check copy' if @debug
-    [1, 2, 3].each do |i|
-      ['b', 'c'].each do |j|
-        assert_equal `du -sb /tmp/host1-sda/bla/bla/test$i 2>&1`.to_i, `du -sb /tmp/host$i-sd$j/bla/bla/test$i 2>&1`.to_i
+    puts 'Check copy' if @@debug
+    [1, 2, 3, 4].each do |i|
+      %w(b c).each do |j|
+        src_size = `du -sb /tmp/host$i-sda/bla/bla/test$i 2>&1`.to_i
+        dst_size = `du -sb /tmp/host$i-sd$j/bla/bla/test$i 2>&1`.to_i
+        assert_equal src_size, dst_size
       end
     end
-    assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
+    assert_equal @@errors_count, FC::Error.where.count, 'new errors in errors table'
     
     @@policy.copies = 2
     @@policy.save
@@ -109,24 +133,41 @@ class DaemonTest < Test::Unit::TestCase
     item_storage.save
     
     sleep 2
-    puts 'Check delete' if @debug
+    puts 'Check delete' if @@debug
     assert_equal 0, `du -sb /tmp/host1-sdc/bla/bla/test1 2>&1`.to_i
-    assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
+    assert_equal @@errors_count, FC::Error.where.count, 'new errors in errors table'
     
-    @item1.mark_deleted
-    FC::ItemStorage.where('item_id = ?', @item2.id).each do |item_storage|
-      item_storage.status = 'error'
-      item_storage.save
+    @item1.immediate_delete
+    @item4.mark_deleted
+    @item2.get_item_storages.each do |ist|
+      ist.status = 'error'
+      ist.save
     end
     @item3.status = 'error'
     @item3.save
     
     sleep 6
-    puts 'Check mark_deleted' if @debug
-    assert_raise(RuntimeError, "Item not deleted after mark_deleted") {@item1.reload}
-    assert_equal 0, FC::ItemStorage.where('item_id = ?', @item2.id).count, "ItemStorages not deleted after status='error'"
+    puts 'Check immediate_delete' if @@debug
+    assert_raise(RuntimeError, 'Item not deleted after mark_deleted') { @item1.reload }
+    assert_equal 0, @item2.get_item_storages.count, "ItemStorages not deleted after status='error'"
     @item3.reload
-    assert_equal 'delete', @item3.status, "ItemStorages not deleted after status='error'"    
-    assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
+    assert_equal 'delete', @item3.status, "ItemStorages not deleted after status='error'"
+    assert_equal @@errors_count, FC::Error.where.count, 'new errors in errors table'
+    assert_equal 'deferred_delete', @item4.status, 'Item not deferred_delete after mark_deleted'
+    assert_same_elements %w(ready ready ready), @item4.get_item_storages.map(&:status)
+
+    sleep 6
+    puts 'Check mark_deleted' if @@debug
+    assert_raise(RuntimeError, 'Item not deleted after mark_deleted') { @item4.reload }
+  end
+
+  private
+
+  def make_file_item(name, tag)
+    FC::Item.create_from_local(@@test_file_path, name, @@policy, :tag => tag)
+  end
+
+  def make_dir_item(name, tag)
+    FC::Item.create_from_local(@@test_dir_path, name, @@policy, :tag => tag)
   end
-end
+end

data/test/db_test.rb

@@ -71,8 +71,9 @@ class DbTest < Test::Unit::TestCase
     assert_nil FC::DB.connect, 'FC::DB.connect is not empty after close call'
 
     FC::DB.stubs(:options_yml_path).returns(db_config_file)
-    FC::DB.connect_by_yml
+    FC::DB.connect_by_yml('test' => 'test')
     assert_true FC::DB.connect.ping, 'Not connected after connect_by_yml'
+    assert_equal FC::DB.options[:test], 'test', 'options keys is not symbolized'
 
     FC::DB.close
     FC::DB.reconnect

data/test/functional_test.rb

@@ -1,7 +1,7 @@
 require 'helper'
 
 class FunctionalTest < Test::Unit::TestCase 
-    class << self
+  class << self
     def startup
       # tmp fake storages dirs
       `rm -rf /tmp/host*-sd*`
@@ -155,4 +155,25 @@ class FunctionalTest < Test::Unit::TestCase
     assert_equal false, File.exists?(tmp_file_path)
     File.unstub(:delete)
   end
+
+  should 'item create_from_local with block for choose storage' do
+    item = FC::Item.create_from_local(@@test_file_path, 
+                                      '/bla/bla/test7', 
+                                      @@policies[0], 
+                                      :tag => 'test') do |storages|
+      storages.select { |s| s.name == 'host2-sda' }
+    end
+    assert_kind_of FC::Item, item
+    assert_equal `du -sb /tmp/host2-sda/bla/bla/test7 2>&1`.to_i, item.size
+    assert_equal 'ready', item.status
+    item_storages = item.get_item_storages
+    assert_equal 1, item_storages.count
+    assert_equal 'ready', item_storages.first.status
+    assert_equal 'host2-sda', item_storages.first.storage_name
+    item = FC::Item.create_from_local(@@test_file_path, 
+                                      '/bla/bla/test8', 
+                                      @@policies[0], 
+                                      :tag => 'test') { [@@storages[4]] }
+    assert_equal 'host3-sda', item.get_item_storages.first.storage_name
+  end
 end

data/test/helper.rb

@@ -1,5 +1,6 @@
 $:.unshift File.expand_path('../lib', File.dirname(__FILE__))
 
+require 'ostruct'
 require "test/unit"
 require "shoulda-context"
 require "filecluster"

data/test/item_test.rb

@@ -16,6 +16,7 @@ class ItemTest < Test::Unit::TestCase
         item_storage
       end
       @@storages << FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :url => 'http://rec3/sda/')
+      @@storages[2].save
     end
     def shutdown
       FC::DB.query("DELETE FROM items_storages")
@@ -24,8 +25,7 @@ class ItemTest < Test::Unit::TestCase
     end
   end
   
-  should "create_from_local" do
-    policy = FC::Policy.new
+  should "create_from_local" do    policy = FC::Policy.new
     assert_raise(ArgumentError) { FC::Item.create_from_local }
     assert_raise(ArgumentError) { FC::Item.create_from_local '/bla/bla' }
     assert_raise(ArgumentError) { FC::Item.create_from_local '/bla/bla', 'test' }
@@ -33,42 +33,50 @@ class ItemTest < Test::Unit::TestCase
     assert_raise() { FC::Item.create_from_local '/bla/bla/bla', 'test', policy}
   end
   
-  should "mark_deleted" do
-    @@item.mark_deleted
+  should "immediate_delete" do
+    @@item.immediate_delete
     @@item.reload
     assert_equal 'delete', @@item.status
     @@item_storages.each do |item_storage| 
       item_storage.reload
       assert_equal 'delete', item_storage.status
+      item_storage.status = 'ready'
+      item_storage.save
+    end
+    @@item.status = 'ready'
+    @@item.save
+  end
+
+  should 'mark_deleted' do    @@item.mark_deleted
+    @@item.reload
+    assert_equal 'deferred_delete', @@item.status
+    @@item_storages.each do |item_storage| 
+      item_storage.reload
+      assert_equal 'ready', item_storage.status
     end
   end
   
-  should "make_item_storage" do
-    storage_size = @@storages[2].size.to_i
+  should "make_item_storage" do    storage_size = @@storages[2].size.to_i
     assert_kind_of FC::ItemStorage, @@item.make_item_storage(@@storages[2])
     assert_equal storage_size+@@item.size, @@storages[2].size
   end
   
-  should "get_item_storages" do
-    assert_same_elements @@item_storages.map(&:id), @@item.get_item_storages.map(&:id)
+  should "get_item_storages" do    assert_same_elements @@item_storages.map(&:id), @@item.get_item_storages.map(&:id)
   end
   
-  should "item get_available_storages" do
-    @@storages.each{|s| s.check_time = 0; s.save}
+  should "item get_available_storages" do    @@storages.each{|s| s.check_time = 0; s.save}
     @@storages[0].update_check_time
     assert_equal 1, @@item.get_available_storages.count
     assert_equal @@storages[0].name, @@item.get_available_storages.first.name
   end
   
-  should "item urls" do
-    @@storages.each{|s| s.check_time = 0; s.save}
+  should "item urls" do    @@storages.each{|s| s.check_time = 0; s.save}
     assert_equal 0, @@item.urls.count
     @@storages.each(&:update_check_time)
     assert_same_elements ["http://rec1/sda/test item", "http://rec2/sda/test item"], @@item.urls
   end
   
-  should "item url by url_weight" do
-    @@storages.each(&:update_check_time)
+  should "item url by url_weight" do    @@storages.each(&:update_check_time)
     @@storages.each{|s| s.url_weight = -1; s.save}
     assert_raise(RuntimeError) { @@item.url }
     

data/test/storage_sync_test.rb

@@ -39,7 +39,9 @@ class StorageSyncTest < Test::Unit::TestCase
     assert_nothing_raised { @item3 = FC::Item.create_from_local(@@test_file_path, 'a/b/c/test3', @@policy, {:tag => 'test'}) }
     assert_nothing_raised { @item4 = FC::Item.create_from_local(@@test_file_path, 'a/b/c/d/test4', @@policy, {:tag => 'test'}) }
     assert_nothing_raised { @item5 = FC::Item.create_from_local(@@test_file_path, 'a/del_file', @@policy, {:tag => 'test'}) }
-    @item5.mark_deleted
+    assert_nothing_raised { @item6 = FC::Item.create_from_local(@@test_file_path, 'a/differed_del_file', @@policy, {:tag => 'test'}) }
+    @item5.immediate_delete
+    @item6.mark_deleted
     `mv /tmp/host1-sda/a/test1 /tmp/host1-sda/test1`
     `mv /tmp/host1-sda/a/b/c/d/test4 /tmp/host1-sda/a/b/c/d/test5`
     `mkdir /tmp/host1-sda/test_dir`
@@ -52,10 +54,12 @@ class StorageSyncTest < Test::Unit::TestCase
     @item2.reload
     @item3.reload
     @item4.reload
+    @item6.reload
     assert_equal 'error', @item1.status
     assert_equal 'ready', @item2.status
     assert_equal 'ready', @item3.status
     assert_equal 'error', @item4.status
+    assert_equal 'deferred_delete', @item6.status
     size = `du -sb #{@@test_file_path} 2>&1`.to_i
     assert_equal 0, `du -sb /tmp/host1-sda/test1 2>&1`.to_i
     assert_equal size, `du -sb /tmp/host1-sda/a/b/test2 2>&1`.to_i
@@ -63,7 +67,8 @@ class StorageSyncTest < Test::Unit::TestCase
     assert_equal 0, `du -sb /tmp/host1-sda/a/b/c/d/test5 2>&1`.to_i
     assert_equal 0, `du -sb /tmp/host1-sda/a/b/c/d 2>&1`.to_i
     assert_equal 0, `du -sb /tmp/host1-sda/test_dir 2>&1`.to_i
-    assert_equal 0, @item5.get_item_storages.size
+    assert_equal 0, @item5.get_item_storages.count
     assert_equal 0, `du -sb /tmp/host1-sda/a/del_file 2>&1`.to_i
+    assert_equal size, `du -sb /tmp/host1-sda/a/differed_del_file 2>&1`.to_i
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - sh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-07 00:00:00.000000000 Z
+date: 2016-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2
@@ -174,7 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.3
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Distributed storage