filecluster 0.2.9 → 0.3.0

data/README.md

@@ -31,7 +31,7 @@ Selecting available storage to store item by policy.create_storages (from left t
     
 ## Copy policy
 
-Selecting available storage to copy item by policy.copy_storages (from left to tight) with the nearest copy_id.
+Selecting available storage to copy item by storage.copy_storages (from left to right).
 
 ## Variables
 

data/lib/daemon/global_daemon_thread.rb

@@ -30,11 +30,11 @@ class GlobalDaemonThread < BaseThread
     all_policies = FC::Policy.where
     
     # policies.get_storages => all_policies.select
-    all_policies.each do |policy|
-      metaclass = class << policy; self; end
+    all_storages.each do |storage|
+      metaclass = class << storage; self; end
       metaclass.send(:define_method, :get_copy_storages) do
         self.class.get_copy_storages_mutex.synchronize do
-          @copy_storages_cache ||= self.copy_storages.split(',').map{|storage_name| all_storages.detect{|s| storage_name == s.name} }
+          @copy_storages_cache ||= self.copy_storages.to_s.split(',').map{|storage_name| all_storages.detect{|s| storage_name == s.name} }
         end
         @copy_storages_cache
       end
@@ -57,7 +57,7 @@ class GlobalDaemonThread < BaseThread
           $log.warn("GlobalDaemonThread: ItemStorage count >= policy.copies for item #{row['item_id']}")
         else
           src_storage = all_storages.detect{|s| item_storages.first == s.name}
-          storage = policy.get_proper_storage_for_copy(row['size'], src_storage.copy_id, item_storages) if src_storage && policy 
+          storage = src_storage.get_proper_storage_for_copy(row['size'], item_storages) if src_storage 
           if storage
             FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
           else

data/lib/fc/db.rb

@@ -85,21 +85,17 @@ module FC
           size bigint NOT NULL DEFAULT 0,
           size_limit bigint NOT NULL DEFAULT 0,
           check_time int DEFAULT NULL,
-          copy_id int NOT NULL DEFAULT 0,
+          copy_storages text NOT NULL DEFAULT '',
           PRIMARY KEY (id), UNIQUE KEY (name), KEY (host)
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
       })
       proc = %{
-        # update policy.create_storages and policy.copy_storages on storage delete and update
+        # update policy.create_storages on storage delete and update
         UPDATE #{@prefix}policies, 
           (SELECT #{@prefix}policies.id, GROUP_CONCAT(#{@prefix}storages.name ORDER BY FIND_IN_SET(#{@prefix}storages.name, create_storages)) as storages FROM #{@prefix}policies LEFT JOIN #{@prefix}storages ON 
-            FIND_IN_SET(#{@prefix}storages.name, create_storages) GROUP BY #{@prefix}policies.id) as policy_create,
-          (SELECT #{@prefix}policies.id, GROUP_CONCAT(#{@prefix}storages.name ORDER BY FIND_IN_SET(#{@prefix}storages.name, copy_storages)) as storages FROM #{@prefix}policies LEFT JOIN #{@prefix}storages ON 
-            FIND_IN_SET(#{@prefix}storages.name, copy_storages) GROUP BY #{@prefix}policies.id) as policy_copy
-        SET 
-          #{@prefix}policies.create_storages = policy_create.storages, 
-          #{@prefix}policies.copy_storages = policy_copy.storages
-        WHERE policy_create.id = #{@prefix}policies.id AND policy_copy.id = #{@prefix}policies.id;
+            FIND_IN_SET(#{@prefix}storages.name, create_storages) GROUP BY #{@prefix}policies.id) as policy_create
+        SET #{@prefix}policies.create_storages = policy_create.storages
+        WHERE policy_create.id = #{@prefix}policies.id;
       }
       proc_update = %{
         IF OLD.name <> NEW.name THEN 
@@ -114,17 +110,14 @@ module FC
           id int NOT NULL AUTO_INCREMENT,
           name varchar(255) NOT NULL DEFAULT '',
           create_storages text NOT NULL DEFAULT '',
-          copy_storages text NOT NULL DEFAULT '',
           copies int NOT NULL DEFAULT 0,
           PRIMARY KEY (id), UNIQUE KEY (name)
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
       })
       proc = %{
-        # update policy.create_storages and policy.copy_storages on policy change - guarantee valid policy.storages
+        # update policy.create_storages on policy change - guarantee valid policy.storages
         SELECT GROUP_CONCAT(name ORDER BY FIND_IN_SET(name, NEW.create_storages)) INTO @create_storages_list FROM #{@prefix}storages WHERE FIND_IN_SET(name, NEW.create_storages);
-        SELECT GROUP_CONCAT(name ORDER BY FIND_IN_SET(name, NEW.copy_storages)) INTO @copy_storages_list FROM #{@prefix}storages WHERE FIND_IN_SET(name, NEW.copy_storages);
         SET NEW.create_storages = @create_storages_list;
-        SET NEW.copy_storages = @copy_storages_list;
       }
       FC::DB.connect.query("CREATE TRIGGER fc_policies_before_insert BEFORE INSERT on #{@prefix}policies FOR EACH ROW BEGIN #{proc} END")
       FC::DB.connect.query("CREATE TRIGGER fc_policies_before_update BEFORE UPDATE on #{@prefix}policies FOR EACH ROW BEGIN #{proc} END")

data/lib/fc/policy.rb

@@ -2,14 +2,13 @@
 
 module FC
   class Policy < DbBase
-    set_table :policies, 'name, create_storages, copy_storages, copies'
+    set_table :policies, 'name, create_storages, copies'
     
     class << self
-      attr_accessor :storages_cache_time, :get_create_storages_mutex, :get_copy_storages_mutex
+      attr_accessor :storages_cache_time, :get_create_storages_mutex
     end
     @storages_cache_time = 20 # ttl for storages cache
     @get_create_storages_mutex = Mutex.new
-    @get_copy_storages_mutex = Mutex.new
     
     def self.filter_by_host(host = nil)
       host = FC::Storage.curr_host unless host
@@ -22,50 +21,18 @@ module FC
       self.class.get_create_storages_mutex.synchronize do
         unless @create_storages_cache && Time.new.to_i - @get_create_storages_time.to_i < self.class.storages_cache_time
           @get_create_storages_time = Time.new.to_i
-          names = create_storages.split(',').map{|s| "'#{s}'"}.join(',')
-          @create_storages_cache = FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
+          names = create_storages.to_s.split(',').map{|s| "'#{s}'"}.join(',')
+          @create_storages_cache = names.empty? ? [] : FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
         end
       end
       @create_storages_cache
     end
     
-    def get_copy_storages
-      self.class.get_copy_storages_mutex.synchronize do
-        unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time
-          @get_copy_storages_time = Time.new.to_i
-          names = copy_storages.split(',').map{|s| "'#{s}'"}.join(',')
-          @copy_storages_cache = FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
-        end
-      end
-      @copy_storages_cache
-    end
-    
     # get available storage for create by size
     def get_proper_storage_for_create(size, exclude = [])
       get_create_storages.detect do |storage|
         !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
       end
     end
-    
-    # get available storage for copy by copy_id and size
-    def get_proper_storage_for_copy(size, copy_id = nil, exclude = [])
-      storages = get_copy_storages
-      storage_index = 0
-      storage_host = nil
-      while s = storages[storage_index]
-        if copy_id.to_i == s.copy_id.to_i && !exclude.include?(s.name)
-          storage_index -= 1 while storage_index > 0 && storages[storage_index-1].host == s.host
-          storage_host = s.host
-          break
-        end
-        storage_index += 1
-      end 
-      storages = (storages[storage_index..-1]+storages[0..storage_index-1]).select do |s|
-        !exclude.include?(s.name) && s.up? && s.size + size < s.size_limit
-      end
-      storage = storages.select{|s| storage_host == s.host}.sort{|a,b| (copy_id.to_i - a.copy_id.to_i).abs <=> (copy_id.to_i - b.copy_id.to_i).abs}.first
-      storage = storages.sort{|a,b| (copy_id.to_i - a.copy_id.to_i).abs <=> (copy_id.to_i - b.copy_id.to_i).abs}.first unless storage
-      storage
-    end
   end
 end

data/lib/fc/storage.rb

@@ -3,12 +3,14 @@ require 'shellwords'
 
 module FC
   class Storage < DbBase
-    set_table :storages, 'name, host, path, url, size, size_limit, check_time, copy_id'
+    set_table :storages, 'name, host, path, url, size, size_limit, check_time, copy_storages'
     
     class << self
-      attr_accessor :check_time_limit
+      attr_accessor :check_time_limit, :storages_cache_time, :get_copy_storages_mutex
     end
     @check_time_limit = 120 # ttl for up status check
+    @storages_cache_time = 20 # ttl for storages cache
+    @get_copy_storages_mutex = Mutex.new
     
     def self.curr_host
       @uname || @uname = `uname -n`.chomp
@@ -24,6 +26,17 @@ module FC
       super params
     end
     
+    def get_copy_storages
+      self.class.get_copy_storages_mutex.synchronize do
+        unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time
+          @get_copy_storages_time = Time.new.to_i
+          names = copy_storages.to_s.split(',').map{|s| "'#{s}'"}.join(',')
+          @copy_storages_cache = names.empty? ? [] : FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
+        end
+      end
+      @copy_storages_cache
+    end
+    
     def update_check_time
       self.check_time = Time.new.to_i
       save
@@ -94,5 +107,12 @@ module FC
       raise r if $?.exitstatus != 0
       r.to_i
     end
+    
+    # get available storage for copy by size
+    def get_proper_storage_for_copy(size, exclude = [])
+      get_copy_storages.select do |storage|
+        !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
+      end.first
+    end
   end
 end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.2.9"
+  VERSION = "0.3.0"
 end

data/lib/manage/policies.rb

@@ -4,7 +4,7 @@ def policies_list
     puts "No storages."
   else
     policies.each do |policy|
-      puts "##{policy.id} #{policy.name}, create storages: #{policy.create_storages}, copy storages: #{policy.copy_storages}, copies: #{policy.copies}"
+      puts "##{policy.id} #{policy.name}, create storages: #{policy.create_storages}, copies: #{policy.copies}"
     end
   end
 end
@@ -16,7 +16,6 @@ def policies_show
   ID:               #{policy.id}
   Name:             #{policy.name}
   Create storages:  #{policy.create_storages}
-  Copy storages:    #{policy.copy_storages}
   Copies:           #{policy.copies}
   Items:            #{count}}
   end
@@ -26,15 +25,13 @@ def policies_add
   puts "Add Policy"
   name = stdin_read_val('Name')
   create_storages = stdin_read_val('Create storages')
-  copy_storages = stdin_read_val('Copy storages')
   copies = stdin_read_val('Copies').to_i
   
   storages = FC::Storage.where.map(&:name)
   create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
-  copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
   
   begin
-    policy = FC::Policy.new(:name => name, :create_storages => create_storages, :copy_storages => copy_storages, :copies => copies)
+    policy = FC::Policy.new(:name => name, :create_storages => create_storages, :copies => copies)
   rescue Exception => e
     puts "Error: #{e.message}"
     exit
@@ -42,7 +39,6 @@ def policies_add
   puts %Q{\nPolicy
   Name:             #{name}
   Create storages:  #{create_storages}
-  Copy storages:    #{copy_storages} 
   Copies:           #{copies}}
   s = Readline.readline("Continue? (y/n) ", false).strip.downcase
   puts ""
@@ -77,22 +73,18 @@ def policies_change
     puts "Change policy ##{policy.id} #{policy.name}"
     name = stdin_read_val("Name (now #{policy.name})", true)
     create_storages = stdin_read_val("Create storages (now #{policy.create_storages})", true)
-    copy_storages = stdin_read_val("Copy storages (now #{policy.copy_storages})", true)
     copies = stdin_read_val("Copies (now #{policy.copies})", true)
     
     storages = FC::Storage.where.map(&:name)
     create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless create_storages.empty?
-    copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
         
     policy.name = name unless name.empty?
     policy.create_storages = create_storages unless create_storages.empty?
-    policy.copy_storages = copy_storages unless copy_storages.empty?
     policy.copies = copies.to_i unless copies.empty?
     
     puts %Q{\nStorage
     Name:             #{policy.name}
     Create storages:  #{policy.create_storages}
-    Copy storages:    #{policy.copy_storages} 
     Copies:           #{policy.copies}}
     s = Readline.readline("Continue? (y/n) ", false).strip.downcase
     puts ""

data/lib/manage/storages.rb

@@ -25,7 +25,7 @@ def storages_show
   Url:            #{storage.url}
   Size:           #{size_to_human storage.size}
   Size limit:     #{size_to_human storage.size_limit}
-  Copy id:        #{storage.copy_id}
+  Copy storages:  #{storage.copy_storages}
   Check time:     #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
   Status:         #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
   Items storages: #{count}}
@@ -39,11 +39,13 @@ def storages_add
   path = stdin_read_val('Path')
   url = stdin_read_val('Url')
   size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
-  copy_id = stdin_read_val('Copy id').to_i
+  copy_storages = stdin_read_val('Copy storages')
+  storages = FC::Storage.where.map(&:name)
+  copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
   begin
     path = path +'/' unless path[-1] == '/'
     path = '/' + path unless path[0] == '/'
-    storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_id => copy_id)
+    storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages)
     print "Calc current size.. "
     size = storage.file_size('', true)
     puts "ok"
@@ -52,13 +54,13 @@ def storages_add
     exit
   end
   puts %Q{\nStorage
-  Name:       #{name}
-  Host:       #{host} 
-  Path:       #{path} 
-  Url:        #{url}
-  Size:       #{size_to_human size}
-  Size limit: #{size_to_human size_limit}
-  Copy id:    #{copy_id}}
+  Name:         #{name}
+  Host:         #{host} 
+  Path:         #{path} 
+  Url:          #{url}
+  Size:         #{size_to_human size}
+  Size limit:   #{size_to_human size_limit}
+  Copy storages #{copy_storages}}
   s = Readline.readline("Continue? (y/n) ", false).strip.downcase
   puts ""
   if s == "y" || s == "yes"
@@ -90,9 +92,11 @@ end
 
 def storages_update_size
   if storage = find_storage
+    FC::DB.close
     print "Calc current size.. "
     size = storage.file_size('', true)
     storage.size = size
+    FC::DB.connect
     begin
       storage.save
     rescue Exception => e
@@ -110,7 +114,7 @@ def storages_change
     path = stdin_read_val("Path (now #{storage.path})", true)
     url = stdin_read_val("Url (now #{storage.url})", true)
     size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
-    copy_id = stdin_read_val("Copy id (now #{storage.copy_id})", true)
+    copy_storages = stdin_read_val("Copy storages (now #{storage.copy_storages})", true)
     
     storage.host = host unless host.empty?
     if !path.empty? && path != storage.path
@@ -123,16 +127,17 @@ def storages_change
     end
     storage.url = url unless url.empty?
     storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
-    storage.copy_id = copy_id.to_i unless copy_id.empty?
+    storages = FC::Storage.where.map(&:name)
+    copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
     
     puts %Q{\nStorage
-    Name:       #{storage.name}
-    Host:       #{storage.host} 
-    Path:       #{storage.path} 
-    Url:        #{storage.url}
-    Size:       #{size_to_human storage.size}
-    Size limit: #{size_to_human storage.size_limit}
-    Copy id:    #{copy_id}}
+    Name:          #{storage.name}
+    Host:          #{storage.host} 
+    Path:          #{storage.path} 
+    Url:           #{storage.url}
+    Size:          #{size_to_human storage.size}
+    Size limit:    #{size_to_human storage.size_limit}
+    Copy storages: #{copy_storages}}
     s = Readline.readline("Continue? (y/n) ", false).strip.downcase
     puts ""
     if s == "y" || s == "yes"
@@ -167,6 +172,7 @@ def storages_sync
     if s == "y" || s == "yes"
       make_storages_sync(storage, true)
       puts "Synchronize done."
+      FC::DB.connect
       storages_update_size
     else
       puts "Canceled."
@@ -183,27 +189,30 @@ def find_storage
   storage
 end
 
-def make_storages_sync(storage, make_delete, silent = false)
+def make_storages_sync(storage, make_delete, silent = false, no_reconnect = false)
   # get all items for storage
+  puts "Getting all items from DB" unless silent
   db_items = {}
-  FC::DB.query("SELECT i.name, ist.id FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}'").each do |row|
+  FC::DB.query("SELECT i.name, ist.id, ist.status FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}'").each do |row|
     name = row['name'].sub(/\/$/, '').sub(/^\//, '').strip
     path = ''
     name.split('/').each do |dir|
       path << dir
-      db_items[path] = [false, path == name ? row['id'].to_i : nil]
+      db_items[path] = [false, path == name ? row['id'].to_i : nil, row['status']]
       path << '/'
     end
   end
+  FC::DB.close unless no_reconnect
   
   # walk on all storage folders and files
+  puts "Getting all files" unless silent
   delete_files = []
   process_storage_dir_sync = lambda do |dir = ''|
     Dir.glob(storage.path+dir+'*').each do |f|
       path = f.sub(storage.path, '')
       if db_items[path]
         db_items[path][0] = true
-        next if db_items[path][1]
+        next if db_items[path][1] && db_items[path][2] != 'delete'
       end
       delete_files << path if File.file?(f)
       process_storage_dir_sync.call(path+'/') if File.directory?(f)
@@ -212,24 +221,28 @@ def make_storages_sync(storage, make_delete, silent = false)
   process_storage_dir_sync.call
     
   # rm delete_files
+  FC::DB.connect unless no_reconnect
   if make_delete
+    puts "Deleting files" unless silent
     delete_files.each do |f|
       # check in DB again
-      next if FC::DB.query("SELECT ist.id FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}' AND i.name='#{f}'").first      
+      next if FC::DB.query("SELECT ist.id FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE ist.item_id = i.id AND ist.storage_name = '#{storage.name}' AND i.name='#{f}' AND ist.status<>'delete'").first      
       path = storage.path+f
       File.delete(path) rescue nil
     end
+    puts "Deleted #{delete_files.count} files" unless silent
   end
-  puts "Deleted #{delete_files.count} files" unless silent
   
   # delete non synchronize items_storages
+  puts "Deleting items from DB" unless silent
   count = 0  
   db_items.values.each do |item|
-    if !item[0] && item[1] 
+    if !item[0] && item[1] || item[2] == 'delete' && item[1]
       count += 1
       FC::DB.query("DELETE FROM #{FC::ItemStorage.table_name} WHERE id=#{item[1]}") if make_delete
     end
   end
+  FC::DB.close unless no_reconnect
   puts "Deleted #{count} items_storages" unless silent
   
   # delete empty folders
@@ -249,9 +262,9 @@ def make_storages_sync(storage, make_delete, silent = false)
   if (ARGV[4])
     File.open(ARGV[4], 'w') do |file|
       db_items.values.each do |item|
-        file.puts item[1] if !item[0] && item[1]
+        file.puts item[1] if !item[0] && item[1] || item[2] == 'delete' && item[1]
       end
     end
     puts "Save deleted items_storages to #{ARGV[4]}" unless silent
   end
-end
+end

data/test/daemon_test.rb

@@ -45,12 +45,12 @@ class DaemonTest < Test::Unit::TestCase
       `cp #{@@test_file_path} #{@@test_dir_path}/bbb/test2`
       
       @@storages = []
-      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_id => 1, :size_limit => 1000000000)
-      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :copy_id => 2, :size_limit => 1000000000)
-      @@storages << FC::Storage.new(:name => 'host1-sdc', :host => 'host1', :path => '/tmp/host1-sdc/', :copy_id => 1, :size_limit => 1000000000)
+      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_storages => 'host1-sdb,host1-sdc', :size_limit => 1000000000)
+      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :copy_storages => 'host1-sda,host1-sdc', :size_limit => 1000000000)
+      @@storages << FC::Storage.new(:name => 'host1-sdc', :host => 'host1', :path => '/tmp/host1-sdc/', :copy_storages => 'host1-sda,host1-sdb', :size_limit => 1000000000)
       @@storages.each { |storage| storage.save}
       
-      @@policy = FC::Policy.new(:create_storages => 'host1-sda,host1-sdb,host1-sdc', :copy_storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2, :name => 'policy 1')
+      @@policy = FC::Policy.new(:create_storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2, :name => 'policy 1')
       @@policy.save
       
       # wait for running fc-daemon
@@ -72,7 +72,7 @@ class DaemonTest < Test::Unit::TestCase
       `rm -rf #{@@test_dir_path}`
     end
   end
-  
+
   should "daemon_all" do
     @@storages.each {|storage| storage.reload}
     assert @@storages[0].up?, "Storage #{@@storages[0].name} down" 

data/test/db_test.rb

@@ -15,9 +15,9 @@ class DbTest < Test::Unit::TestCase
       @@storages_ids = storages.map{|storage| storage.save; storage.id }
       
       policies = []
-      policies << FC::Policy.new(:create_storages => 'rec1-sda,rec1-sdd', :copy_storages => 'rec2-sda,rec2-sdd', :copies => 2, :name => 'policy 1')
-      policies << FC::Policy.new(:create_storages => 'rec1-sda,bla,rec2-sdd', :copy_storages => 'rec2-sda,bla,rec1-sdd', :copies => 3, :name => 'policy 2')
-      policies << FC::Policy.new(:create_storages => 'bla,rec1-sda,test', :copy_storages => 'bla,rec2-sda,test', :copies => 4, :name => 'policy 3')
+      policies << FC::Policy.new(:create_storages => 'rec1-sda,rec1-sdd', :copies => 2, :name => 'policy 1')
+      policies << FC::Policy.new(:create_storages => 'rec1-sda,bla,rec2-sdd', :copies => 3, :name => 'policy 2')
+      policies << FC::Policy.new(:create_storages => 'bla,rec1-sda,test', :copies => 4, :name => 'policy 3')
       @@policies_ids = policies.map{|policy| policy.save; policy.id }
       
       items = []
@@ -78,13 +78,9 @@ class DbTest < Test::Unit::TestCase
     assert_equal 'rec1-sda,rec1-sdd', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
     assert_equal 'rec1-sda,rec2-sdd', @policies[1].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
     assert_equal 'rec1-sda', @policies[2].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
-    assert_equal 'rec2-sda,rec2-sdd', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
-    assert_equal 'rec2-sda,rec1-sdd', @policies[1].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
-    assert_equal 'rec2-sda', @policies[2].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
     
-    assert_raise(Mysql2::Error, 'Create policy with uniq name') { FC::Policy.new(:create_storages => 'bla,test', :copy_storages => 'rec2-sda', :name => 'policy 1').save }
-    assert_raise(Mysql2::Error, 'Create policy with incorrect create_storages') { FC::Policy.new(:create_storages => 'bla,test', :copy_storages => 'rec2-sda', :name => 'new policy').save }
-    assert_raise(Mysql2::Error, 'Create policy with incorrect copy_storages') { FC::Policy.new(:create_storages => 'rec1-sda', :copy_storages => 'bla,test', :name => 'new policy').save }
+    assert_raise(Mysql2::Error, 'Create policy with uniq name') { FC::Policy.new(:create_storages => 'bla,test', :name => 'policy 1').save }
+    assert_raise(Mysql2::Error, 'Create policy with incorrect create_storages') { FC::Policy.new(:create_storages => 'bla,test', :name => 'new policy').save }
     
     assert_raise(Mysql2::Error, 'Change storage name with linked polices') { @storages[0].name = 'blabla'; @storages[0].save }
     assert_raise(Mysql2::Error, 'Delete storage name with linked polices') { @storages[0].delete }
@@ -94,19 +90,15 @@ class DbTest < Test::Unit::TestCase
     @policies[0].reload
     @policies[1].reload
     assert_equal 'rec1-sda', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages after storage change"
-    assert_equal 'rec2-sda', @policies[1].copy_storages, "Policy (id=#{@policies[1].id}) incorrect copy_storages after storage change"
     @storages[7].delete  #rec2-sdd
     @@storages_ids.delete(@storages[7].id)
     @policies[0].reload
     @policies[1].reload
     assert_equal 'rec1-sda', @policies[1].create_storages, "Policy (id=#{@policies[1].id}) incorrect create_storages after storage delete"
-    assert_equal 'rec2-sda', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages after storage delete"
     @policies[0].create_storages = 'rec2-sda,rec1-sda,bla bla'
-    @policies[0].copy_storages = 'rec1-sdb,rec2-sda,bla bla'
     @policies[0].save
     @policies[0].reload
     assert_equal 'rec2-sda,rec1-sda', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages after change"
-    assert_equal 'rec1-sdb,rec2-sda', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages after change"
     assert_raise(Mysql2::Error, 'Save empty policy storage') { @policies[0].create_storages = 'blabla'; @policies[0].save }
   end
   

data/test/policy_test.rb

@@ -4,14 +4,14 @@ class PolicyTest < Test::Unit::TestCase
   class << self
     def startup
       @@storages = []
-      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :copy_id => 1, :size_limit => 10)
-      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_id => 2 , :size_limit => 100)
-      @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :copy_id => 4 , :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :size_limit => 10)
+      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :size_limit => 100)
       @@storages.each {|storage| storage.save}
-      @@storage3 = FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_id => 3 , :size_limit => 10)
+      @@storage3 = FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :size_limit => 10)
       @@storage3.save
             
-      @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 1')
+      @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copies => 1, :name => 'policy 1')
       @@policy.save
     end
     def shutdown
@@ -30,18 +30,6 @@ class PolicyTest < Test::Unit::TestCase
     assert_equal 2, @@policy.get_create_storages.size
   end
   
-  should "get_copy_storages" do
-    @@policy.copy_storages = 'rec1-sda,rec2-sda,rec2-sdb'
-    @@policy.save
-    FC::Policy.storages_cache_time = 10
-    assert_same_elements @@storages.map(&:id), @@policy.get_copy_storages.map(&:id)
-    @@policy.copy_storages = 'rec1-sda,rec2-sda'
-    @@policy.save
-    assert_equal @@storages.size, @@policy.get_copy_storages.size
-    FC::Policy.storages_cache_time = 0
-    assert_equal 2, @@policy.get_copy_storages.size
-  end
-  
   should "get_proper_storage_for_create" do
     @@storages.each {|storage| storage.check_time = 0; storage.save}
     @@storage3.check_time = 0
@@ -56,40 +44,6 @@ class PolicyTest < Test::Unit::TestCase
     assert_nil @@policy.get_proper_storage_for_create(1000), 'all storages full'
   end
   
-  should "get_proper_storage_for_copy" do
-    @@storages.each {|storage| storage.check_time = 0; storage.save}
-    FC::Policy.storages_cache_time = 0
-    assert_nil @@policy.get_proper_storage_for_copy(1), 'all storages down'
-    @@storages[0].update_check_time
-    @@storage3.update_check_time
-    assert_equal @@storages[0].id, @@policy.get_proper_storage_for_copy(5).id, 'first storages up'
-    assert_nil @@policy.get_proper_storage_for_copy(20), 'first storage full'
-    @@storages[1].update_check_time
-    assert_equal @@storages[1].id, @@policy.get_proper_storage_for_copy(20).id, 'second storages up'
-    assert_nil @@policy.get_proper_storage_for_copy(1000), 'all storages full'
-    
-    @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda'
-    @@policy.save
-    assert_equal 'rec1-sda', @@policy.get_proper_storage_for_copy(1).name, 'storage with min copy_id in copy_storages'
-    assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 2).name, 'storage by copy_id'
-    @@policy.copy_storages = 'rec1-sda,rec3-sda'
-    @@policy.save
-    assert_equal 'rec1-sda', @@policy.get_proper_storage_for_copy(1, 2).name, 'storage by copy_id'
-    @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda,rec2-sdb'
-    @@policy.save
-    assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
-    @@policy.copy_storages = 'rec2-sda,rec3-sda,rec1-sda,rec2-sdb'
-    @@policy.save
-    @@storages[2].update_check_time
-    assert_equal 'rec2-sdb', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
-    
-    @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda'
-    @@policy.save
-    @@storages[0].check_time = 0
-    @@storages[0].save
-    assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 1).name, 'storage by copy_id'
-  end
-  
   should "filter_by_host" do
     FC::Storage.stubs(:curr_host).returns('rec2')
     FC::Policy.new(:create_storages => 'rec3-sda,rec2-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 2').save

data/test/{storage_sync.rb → storage_sync_test.rb}

@@ -1,7 +1,7 @@
 require 'helper'
 require 'manage'
 
-class DaemonTest < Test::Unit::TestCase
+class StorageSyncTest < Test::Unit::TestCase
   class << self
     def startup
       # tmp fake storage dir
@@ -12,9 +12,9 @@ class DaemonTest < Test::Unit::TestCase
       @@test_file_path = '/tmp/fc_test_file'
       `dd if=/dev/urandom of=#{@@test_file_path} bs=1M count=1 2>&1`
       
-      @@storage = FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_id => 1, :size_limit => 1000000000, :check_time => Time.new.to_i)
+      @@storage = FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size_limit => 1000000000, :check_time => Time.new.to_i)
       @@storage.save
-      @@policy = FC::Policy.new(:create_storages => 'host1-sda', :copy_storages => 'host1-sda', :copies => 1, :name => 'policy 1')
+      @@policy = FC::Policy.new(:create_storages => 'host1-sda', :copies => 1, :name => 'policy 1')
       @@policy.save
     end
     
@@ -38,13 +38,15 @@ class DaemonTest < Test::Unit::TestCase
     assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'a/b/test2', @@policy, {:tag => 'test'}) }
     assert_nothing_raised { @item3 = FC::Item.create_from_local(@@test_file_path, 'a/b/c/test3', @@policy, {:tag => 'test'}) }
     assert_nothing_raised { @item4 = FC::Item.create_from_local(@@test_file_path, 'a/b/c/d/test4', @@policy, {:tag => 'test'}) }
+    assert_nothing_raised { @item5 = FC::Item.create_from_local(@@test_file_path, 'a/del_file', @@policy, {:tag => 'test'}) }
+    @item5.mark_deleted
     `mv /tmp/host1-sda/a/test1 /tmp/host1-sda/test1`
     `mv /tmp/host1-sda/a/b/c/d/test4 /tmp/host1-sda/a/b/c/d/test5`
     `mkdir /tmp/host1-sda/test_dir`
     `cp #{@@test_file_path} /tmp/host1-sda/test_dir/t1`
     `cp #{@@test_file_path} /tmp/host1-sda/test_dir/t2`
     
-    make_storages_sync(@@storage, true, true)
+    make_storages_sync(@@storage, true, true, true)
     
     @item1.reload
     @item2.reload
@@ -60,6 +62,8 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal size, `du -sb /tmp/host1-sda/a/b/c/test3 2>&1`.to_i
     assert_equal 0, `du -sb /tmp/host1-sda/a/b/c/d/test5 2>&1`.to_i
     assert_equal 0, `du -sb /tmp/host1-sda/a/b/c/d 2>&1`.to_i
-    assert_equal 0, `du -sb tmp/host1-sda/test_dir 2>&1`.to_i
+    assert_equal 0, `du -sb /tmp/host1-sda/test_dir 2>&1`.to_i
+    assert_equal 0, @item5.get_item_storages.size
+    assert_equal 0, `du -sb /tmp/host1-sda/a/del_file 2>&1`.to_i
   end
 end

data/test/storage_test.rb

@@ -1,6 +1,24 @@
 require 'helper'
 
 class StorageTest < Test::Unit::TestCase
+  class << self
+    def startup
+      @@storages = []
+      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :copy_storages => 'rec2-sda,rec3-sda', :size_limit => 10)
+      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_storages => 'rec1-sda,rec3-sda', :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_storages => 'rec1-sda,rec2-sda', :size_limit => 10)
+      @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :size_limit => 100)
+      @@storages.each {|storage| storage.save}
+      
+      @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copies => 1, :name => 'policy 1')
+      @@policy.save
+    end
+    def shutdown
+      FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM storages")
+    end
+  end 
+  
   should "curr_host" do
     assert ! FC::Storage.curr_host.to_s.empty?
   end
@@ -12,7 +30,7 @@ class StorageTest < Test::Unit::TestCase
   end
   
   should "update_check_time and up?" do
-    storage = FC::Storage.new(:name => 'rec1-sda', :host => 'rec1')
+    storage = FC::Storage.new(:name => 'rec1-test', :host => 'rec1')
     storage.save
     assert_equal false, storage.up?
     storage.update_check_time
@@ -25,4 +43,30 @@ class StorageTest < Test::Unit::TestCase
     storage.delete
   end
   
+  should "get_copy_storages" do
+    @@storages[2].copy_storages = 'rec1-sda,rec2-sda,rec2-sdb'
+    @@storages[2].save
+    FC::Storage.storages_cache_time = 10
+    assert_equal 'rec1-sda,rec2-sda,rec2-sdb', @@storages[2].get_copy_storages.map(&:name).join(',')
+    @@storages[2].copy_storages = 'rec1-sda,rec2-sda'
+    @@storages[2].save
+    assert_equal 'rec1-sda,rec2-sda,rec2-sdb', @@storages[2].get_copy_storages.map(&:name).join(',')
+    FC::Storage.storages_cache_time = 0
+    assert_equal 'rec1-sda,rec2-sda', @@storages[2].get_copy_storages.map(&:name).join(',')
+  end
+  
+  should "get_proper_storage_for_copy" do
+    @@storages.each {|storage| storage.check_time = 0; storage.save}
+    @@storages[3].update_check_time
+    assert_nil @@storages[3].get_proper_storage_for_copy(1), 'empty copy_storages'
+    assert_nil @@storages[2].get_proper_storage_for_copy(1), 'all storages down'
+
+    @@storages[0].update_check_time
+    assert_equal 'rec1-sda', @@storages[2].get_proper_storage_for_copy(5).name, 'first storages up'
+    assert_nil @@storages[2].get_proper_storage_for_copy(20), 'first storage full'
+    @@storages[1].update_check_time
+    assert_equal 'rec1-sda', @@storages[2].get_proper_storage_for_copy(5).name, 'second storages up, small file'
+    assert_equal 'rec2-sda', @@storages[2].get_proper_storage_for_copy(20).name, 'second storages up, big file'
+    assert_nil @@storages[2].get_proper_storage_for_copy(1000), 'second storages up, huge file'
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-11 00:00:00.000000000 Z
+date: 2013-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -174,7 +174,7 @@ files:
 - test/helper.rb
 - test/item_test.rb
 - test/policy_test.rb
-- test/storage_sync.rb
+- test/storage_sync_test.rb
 - test/storage_test.rb
 - test/var_test.rb
 - test/version_test.rb
@@ -192,7 +192,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3823121111064236338
+      hash: 2037414088552914900
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -201,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3823121111064236338
+      hash: 2037414088552914900
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -217,7 +217,7 @@ test_files:
 - test/helper.rb
 - test/item_test.rb
 - test/policy_test.rb
-- test/storage_sync.rb
+- test/storage_sync_test.rb
 - test/storage_test.rb
 - test/var_test.rb
 - test/version_test.rb