filecluster 0.0.8 → 0.0.9

data/README.md

@@ -1,12 +1,21 @@
-# Dstorage
+# FileCluster
+
+A set of scripts to manage files on multiple dc/host/drive.
+
+Item (storage unit)- file or folder.
+Storage - folder, usually separate drive.
+Policy - rule for selecting storage to store the item and сreate copies.
+
+Daemon monitors the availability of each storage and copy files between them.
+
+If the storage is not available or ended the available space is used another available storage according to the policy.
 
-TODO: Write a gem description
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
-    gem 'dstorage'
+    gem 'filecluster'
 
 And then execute:
 
@@ -14,7 +23,15 @@ And then execute:
 
 Or install it yourself as:
 
-    $ gem install dstorage
+    $ gem install filecluster
+
+## Create policy
+
+Selecting available storage to store item by policy.create_storages (from left to tight).
+    
+## Copy policy
+
+Selecting available storage to copy item by policy.copy_storages (from left to tight) with the nearest copy_id.
 
 ## Usage
 

data/TODO

@@ -1,3 +1,6 @@
+периодически удалять пустые папки
+при создании item-а по папке проверять что папка не пустая
+проработать item-ы папки
 добавить в check storage проверку на дорступность урла
 
 алерт на доступность в каждоый политике стораджа на запись

data/bin/fc-manage

@@ -18,19 +18,22 @@ commands_help = {
     'show list and manage storages', 
     %q{Usage: fc-manage [options] storages <command>
 Command:
-   list        show all stotages for all hosts
-   show <name> show full info for storage   
-   add         add new storage
-   rm <name>   delete storage
+   list                show all stotages for all hosts
+   show <name>         show full info for storage   
+   add                 add new storage
+   rm <name>           delete storage
+   change <name>       change storage attributes
+   update_size <name>  update storage size
 }],
   'policies' => [
     'show list and manage plicies', 
     %q{Usage: fc-manage [options] plicies <command>
 Command:
-   list        show all plicies
-   show <id>   show full info for policy   
-   add         add new policy
-   rm <id>     delete policy
+   list                show all plicies
+   show <id/name>      show full info for policy   
+   add                 add new policy
+   rm <id/name>        delete policy
+   change <id/name>    change policy attributes
 }],
   'show' => [
     'show variable', 

data/lib/daemon/global_daemon_thread.rb

@@ -32,13 +32,12 @@ class GlobalDaemonThread < BaseThread
     # policies.get_storages => all_policies.select
     all_policies.each do |policy|
       metaclass = class << policy; self; end
-      metaclass.send(:define_method, :get_storages) do
-        policy_storages = self.storages.split(',')
-        all_storages.select{|storage| policy_storages.member?(storage.name)}
+      metaclass.send(:define_method, :get_copy_storages) do
+        @copy_storages_cache ||= self.copy_storages.split(',').map{|storage_name| all_storages.detect{|s| storage_name == s.name} }
       end
     end
     
-    sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name) as storages, p.id as policy_id, p.copies as policy_copies "+
+    sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name ORDER BY ist.id) as storages, p.id as policy_id, p.copies as policy_copies "+
       "FROM #{FC::Item.table_name} as i, #{FC::Policy.table_name} as p, #{FC::ItemStorage.table_name} as ist WHERE "+
       "i.policy_id = p.id AND ist.item_id = i.id AND i.copies > 0 AND i.copies < p.copies AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT 1000"
     r = FC::DB.connect.query(sql)
@@ -50,8 +49,9 @@ class GlobalDaemonThread < BaseThread
       elsif item_storages.size >= row['policy_copies']
         $log.warn("GlobalDaemonThread: ItemStorage count >= policy.copies for item #{row['item_id']}")
       else
+        src_storage = all_storages.detect{|s| item_storages.first == s.name}
         policy = all_policies.detect{|p| row['policy_id'] == p.id}
-        storage = policy.get_proper_storage(row['size'], item_storages) if policy
+        storage = policy.get_proper_storage_for_copy(row['size'], src_storage.copy_id, item_storages) if src_storage && policy 
         error 'No available storage', :item_id => row['item_id'] unless storage
         FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
       end

data/lib/fc/db.rb

@@ -71,14 +71,21 @@ module FC
           size bigint NOT NULL DEFAULT 0,
           size_limit bigint NOT NULL DEFAULT 0,
           check_time int DEFAULT NULL,
+          copy_id int NOT NULL DEFAULT 0,
           PRIMARY KEY (id), UNIQUE KEY (name), KEY (host)
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
       })
       proc = %{
-        # update policy.storages on storage delete and update
+        # update policy.create_storages and policy.copy_storages on storage delete and update
         UPDATE #{@prefix}policies, 
-          (SELECT #{@prefix}policies.id, GROUP_CONCAT(name) as storages FROM #{@prefix}policies LEFT JOIN #{@prefix}storages ON FIND_IN_SET(name, storages) GROUP BY #{@prefix}policies.id) as new_policy
-        SET #{@prefix}policies.storages = new_policy.storages WHERE #{@prefix}policies.id = new_policy.id;
+          (SELECT #{@prefix}policies.id, GROUP_CONCAT(#{@prefix}storages.name ORDER BY FIND_IN_SET(#{@prefix}storages.name, create_storages)) as storages FROM #{@prefix}policies LEFT JOIN #{@prefix}storages ON 
+            FIND_IN_SET(#{@prefix}storages.name, create_storages) GROUP BY #{@prefix}policies.id) as policy_create,
+          (SELECT #{@prefix}policies.id, GROUP_CONCAT(#{@prefix}storages.name ORDER BY FIND_IN_SET(#{@prefix}storages.name, copy_storages)) as storages FROM #{@prefix}policies LEFT JOIN #{@prefix}storages ON 
+            FIND_IN_SET(#{@prefix}storages.name, copy_storages) GROUP BY #{@prefix}policies.id) as policy_copy
+        SET 
+          #{@prefix}policies.create_storages = policy_create.storages, 
+          #{@prefix}policies.copy_storages = policy_copy.storages
+        WHERE policy_create.id = #{@prefix}policies.id AND policy_copy.id = #{@prefix}policies.id;
       }
       proc_update = %{
         IF OLD.name <> NEW.name THEN 
@@ -91,15 +98,19 @@ module FC
       FC::DB.connect.query(%{
         CREATE TABLE #{@prefix}policies (
           id int NOT NULL AUTO_INCREMENT,
-          storages text NOT NULL DEFAULT '',
+          name varchar(255) NOT NULL DEFAULT '',
+          create_storages text NOT NULL DEFAULT '',
+          copy_storages text NOT NULL DEFAULT '',
           copies int NOT NULL DEFAULT 0,
-          PRIMARY KEY (id)
+          PRIMARY KEY (id), UNIQUE KEY (name)
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
       })
       proc = %{
-        # update policy.storages on policy change - guarantee valid policy.storages
-        SELECT GROUP_CONCAT(name) INTO @storages_list FROM #{@prefix}storages WHERE FIND_IN_SET(name, NEW.storages);
-        SET NEW.storages = @storages_list;
+        # update policy.create_storages and policy.copy_storages on policy change - guarantee valid policy.storages
+        SELECT GROUP_CONCAT(name ORDER BY FIND_IN_SET(name, NEW.create_storages)) INTO @create_storages_list FROM #{@prefix}storages WHERE FIND_IN_SET(name, NEW.create_storages);
+        SELECT GROUP_CONCAT(name ORDER BY FIND_IN_SET(name, NEW.copy_storages)) INTO @copy_storages_list FROM #{@prefix}storages WHERE FIND_IN_SET(name, NEW.copy_storages);
+        SET NEW.create_storages = @create_storages_list;
+        SET NEW.copy_storages = @copy_storages_list;
       }
       FC::DB.connect.query("CREATE TRIGGER fc_policies_before_insert BEFORE INSERT on #{@prefix}policies FOR EACH ROW BEGIN #{proc} END")
       FC::DB.connect.query("CREATE TRIGGER fc_policies_before_update BEFORE UPDATE on #{@prefix}policies FOR EACH ROW BEGIN #{proc} END")

data/lib/fc/item.rb

@@ -23,7 +23,7 @@ module FC
       raise 'Zero size path' if item_params[:size] == 0
       
       if local_path.include?(item_name)
-        storage = policy.get_storages.detect{|s| local_path.index(s.path) == 0 && local_path.sub(s.path, '') == item_params[:name]}
+        storage = policy.get_create_storages.detect{|s| local_path.index(s.path) == 0 && local_path.sub(s.path, '') == item_params[:name]}
         FC::Error.raise "local_path #{local_path} is not valid path for policy ##{policy.id}" unless storage
       end
       
@@ -47,7 +47,7 @@ module FC
         item_storage = item.make_item_storage(storage, 'ready')
         item.reload
       else 
-        storage = policy.get_proper_storage(item.size)
+        storage = policy.get_proper_storage_for_create(item.size)
         FC::Error.raise 'No available storage', :item_id => item.id unless storage
         item_storage = item.make_item_storage(storage)
         item.copy_item_storage(local_path, storage, item_storage)

data/lib/fc/policy.rb

@@ -2,24 +2,43 @@
 
 module FC
   class Policy < DbBase
-    set_table :policies, 'storages, copies'
+    set_table :policies, 'name, create_storages, copy_storages, copies'
     
     class << self
       attr_accessor :storages_cache_time
     end
     @storages_cache_time = 20 # ttl for storages cache
     
-    def get_storages
-      return @storages_cache if @storages_cache && Time.new.to_i - @get_storages_time.to_i < self.class.storages_cache_time
-      @get_storages_time = Time.new.to_i
-      @storages_cache = FC::Storage.where("name IN (#{storages.split(',').map{|s| "'#{s}'"}.join(',')})")
+    def get_create_storages
+      return @create_storages_cache if @create_storages_cache && Time.new.to_i - @get_create_storages_time.to_i < self.class.storages_cache_time
+      @get_create_storages_time = Time.new.to_i
+      names = create_storages.split(',').map{|s| "'#{s}'"}.join(',')
+      @create_storages_cache = FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
     end
     
-    # get available storage for object by size
-    def get_proper_storage(size, exclude = [])
-      get_storages.detect do |storage|
+    def get_copy_storages
+      return @copy_storages_cache if @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time
+      @get_copy_storages_time = Time.new.to_i
+      names = copy_storages.split(',').map{|s| "'#{s}'"}.join(',')
+      @copy_storages_cache = FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
+    end
+    
+    # get available storage for create by size
+    def get_proper_storage_for_create(size, exclude = [])
+      get_create_storages.detect do |storage|
+        !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
+      end
+    end
+    
+    # get available storage for copy by copy_id and size
+    def get_proper_storage_for_copy(size, copy_id = nil, exclude = [])
+      storages = get_copy_storages.select do |storage|
         !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
       end
+      storage = storages.detect{|s| copy_id.to_i == s.copy_id.to_i}
+      storage = storages.detect{|s| copy_id.to_i < s.copy_id.to_i} unless storage
+      storage = storages.first unless storage
+      storage
     end
   end
 end

data/lib/fc/storage.rb

@@ -2,7 +2,7 @@
 
 module FC
   class Storage < DbBase
-    set_table :storages, 'name, host, path, url, size, size_limit, check_time'
+    set_table :storages, 'name, host, path, url, size, size_limit, check_time, copy_id'
     
     class << self
       attr_accessor :check_time_limit
@@ -16,7 +16,9 @@ module FC
     def initialize(params = {})
       path = (params['path'] || params[:path])
       if path && !path.to_s.empty?
+        path += '/' unless path[-1] == '/'
         raise "Storage path must be like '/bla/bla../'" unless path.match(/^\/.*\/$/)
+        params['path'] = params[:path] = path
       end
       super params
     end
@@ -31,47 +33,52 @@ module FC
     end
     
     def up?
-      Time.new.to_i - check_time.to_i < self.class.check_time_limit
+      check_time_delay < self.class.check_time_limit
     end
     
     # copy local_path to storage
     def copy_path(local_path, file_name)
-      cmd = self.class.curr_host == host ? 
-        "cp -r #{local_path} #{self.path}#{file_name}" : 
-        "scp -rB #{local_path} #{self.host}:#{self.path}#{file_name}"
+      dst_path = "#{self.path}#{file_name}"
+      
+      cmd = self.class.curr_host == host ? "mkdir -p #{File.dirname(dst_path)}" : "ssh -oBatchMode=yes #{self.host} 'mkdir -p #{File.dirname(dst_path)}'"
+      r = `#{cmd} 2>&1`
+      raise r if $?.exitstatus != 0
+      
+      cmd = self.class.curr_host == host ? "cp -r #{local_path} #{dst_path}" : "scp -rB #{local_path} #{self.host}:#{dst_path}"
       r = `#{cmd} 2>&1`
       raise r if $?.exitstatus != 0
     end
     
     # copy object to local_path
     def copy_to_local(file_name, local_path)
-      cmd = self.class.curr_host == host ? 
-        "cp -r #{self.path}#{file_name} #{local_path}" : 
-        "scp -rB #{self.host}:#{self.path}#{file_name} #{local_path}"
+      src_path = "#{self.path}#{file_name}"
+      
+      cmd = self.class.curr_host == host ? "mkdir -p #{File.dirname(local_path)}" : "ssh -oBatchMode=yes #{self.host} 'mkdir -p #{File.dirname(local_path)}'"
+      r = `#{cmd} 2>&1`
+      raise r if $?.exitstatus != 0
+      
+      cmd = self.class.curr_host == host ? "cp -r #{src_path} #{local_path}" : "scp -rB #{self.host}:#{src_path} #{local_path}"
       r = `#{cmd} 2>&1`
       raise r if $?.exitstatus != 0
     end
     
     # delete object from storage
     def delete_file(file_name)
-      cmd = self.class.curr_host == host ? 
-        "rm -rf #{self.path}#{file_name}" : 
-        "ssh -oBatchMode=yes #{self.host} 'rm -rf #{self.path}#{file_name}'"
+      dst_path = "#{self.path}#{file_name}"
+      cmd = self.class.curr_host == host ? "rm -rf #{dst_path}" : "ssh -oBatchMode=yes #{self.host} 'rm -rf #{dst_path}'"
       r = `#{cmd} 2>&1`
       raise r if $?.exitstatus != 0
       
-      cmd = self.class.curr_host == host ? 
-        "ls -la #{self.path}#{file_name}" : 
-        "ssh -oBatchMode=yes #{self.host} 'ls -la #{self.path}#{file_name}'"
+      cmd = self.class.curr_host == host ? "ls -la #{dst_path}" : "ssh -oBatchMode=yes #{self.host} 'ls -la #{dst_path}'"
       r = `#{cmd} 2>/dev/null`
-      raise "Path #{self.path}#{file_name} not deleted" unless r.empty?
+      raise "Path #{dst_path} not deleted" unless r.empty?
     end
     
     # return object size on storage
     def file_size(file_name)
-      cmd = self.class.curr_host == host ? 
-        "du -sb #{self.path}#{file_name}" : 
-        "ssh -oBatchMode=yes #{self.host} 'du -sb #{self.path}#{file_name}'"
+      dst_path = "#{self.path}#{file_name}"
+      
+      cmd = self.class.curr_host == host ? "du -sb #{dst_path}" : "ssh -oBatchMode=yes #{self.host} 'du -sb #{dst_path}'"
       r = `#{cmd} 2>&1`
       raise r if $?.exitstatus != 0
       r.to_i

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/manage/policies.rb

@@ -4,39 +4,46 @@ def policies_list
     puts "No storages."
   else
     policies.each do |policy|
-      puts "##{policy.id} storages: #{policy.storages}, copies: #{policy.copies}"
+      puts "##{policy.id} #{policy.name}, create storages: #{policy.create_storages}, copy storages: #{policy.copy_storages}, copies: #{policy.copies}"
     end
   end
 end
 
 def policies_show
-  id = ARGV[2]
-  policy = FC::Policy.where('id = ?', id).first
-  if !policy
-    puts "Policy ##{id} not found."
-  else
+  if policy = find_policy
     count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::Item.table_name} WHERE policy_id = #{policy.id}").first['cnt']
     puts %Q{Policy
-  ID:         #{policy.id}
-  Storages:   #{policy.storages}
-  Copies:     #{policy.copies}
-  Items:      #{count}}
+  ID:               #{policy.id}
+  Name:             #{policy.name}
+  Create storages:  #{policy.create_storages}
+  Copy storages:    #{policy.copy_storages}
+  Copies:           #{policy.copies}
+  Items:            #{count}}
   end
 end
 
 def policies_add
   puts "Add Policy"
-  storages = stdin_read_val('Storages')
+  name = stdin_read_val('Name')
+  create_storages = stdin_read_val('Create storages')
+  copy_storages = stdin_read_val('Copy storages')
   copies = stdin_read_val('Copies').to_i
+  
+  storages = FC::Storage.where.map(&:name)
+  create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
+  copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
+  
   begin
-    policy = FC::Policy.new(:storages => storages, :copies => copies)
+    policy = FC::Policy.new(:name => name, :create_storages => create_storages, :copy_storages => copy_storages, :copies => copies)
   rescue Exception => e
     puts "Error: #{e.message}"
     exit
   end
   puts %Q{\nPolicy
-  Storages:   #{storages} 
-  Copies:     #{copies}}
+  Name:             #{name}
+  Create storages:  #{create_storages}
+  Copy storages:    #{copy_storages} 
+  Copies:           #{copies}}
   s = Readline.readline("Continue? (y/n) ", false).strip.downcase
   puts ""
   if s == "y" || s == "yes"
@@ -53,11 +60,7 @@ def policies_add
 end
 
 def policies_rm
-  id = ARGV[2]
-  policy = FC::Policy.where('id = ?', id).first
-  if !policy
-    puts "Policy ##{id} not found."
-  else
+  if policy = find_policy
     s = Readline.readline("Continue? (y/n) ", false).strip.downcase
     puts ""
     if s == "y" || s == "yes"
@@ -68,3 +71,50 @@ def policies_rm
     end
   end
 end
+
+def policies_change
+  if policy = find_policy
+    puts "Change policy ##{policy.id} #{policy.name}"
+    name = stdin_read_val("Name (now #{policy.name})", true)
+    create_storages = stdin_read_val("Create storages (now #{policy.create_storages})", true)
+    copy_storages = stdin_read_val("Copy storages (now #{policy.copy_storages})", true)
+    copies = stdin_read_val("Copies (now #{policy.copies})", true)
+    
+    storages = FC::Storage.where.map(&:name)
+    create_storages = create_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless create_storages.empty?
+    copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
+        
+    policy.name = name unless name.empty?
+    policy.create_storages = name unless create_storages.empty?
+    policy.copy_storages = name unless copy_storages.empty?
+    policy.copies = copies.to_i unless copies.empty?
+    
+    puts %Q{\nStorage
+    Name:             #{policy.name}
+    Create storages:  #{policy.create_storages}
+    Copy storages:    #{policy.copy_storages} 
+    Copies:           #{policy.copies}}
+    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+    puts ""
+    if s == "y" || s == "yes"
+      begin
+        policy.save
+      rescue Exception => e
+        puts "Error: #{e.message}"
+        exit
+      end
+      puts "ok"
+    else
+      puts "Canceled."
+    end
+  end
+end
+
+private
+
+def find_policy
+  policy = FC::Policy.where('id = ?', ARGV[2]).first
+  policy = FC::Policy.where('name = ?', ARGV[2]).first unless policy
+  puts "Policy #{ARGV[2]} not found." unless policy
+  policy
+end

data/lib/manage/storages.rb

@@ -13,11 +13,7 @@ def storages_list
 end
 
 def storages_show
-  name = ARGV[2]
-  storage = FC::Storage.where('name = ?', name).first
-  if !storage
-    puts "Storage #{name} not found."
-  else
+  if storage = find_storage
     count = FC::DB.connect.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
     puts %Q{Storage
   Name:           #{storage.name}
@@ -26,6 +22,7 @@ def storages_show
   Url:            #{storage.url}
   Size:           #{size_to_human storage.size}
   Size limit:     #{size_to_human storage.size_limit}
+  Copy id:        #{storage.copy_id}
   Check time:     #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
   Status:         #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
   Items storages: #{count}}
@@ -39,8 +36,11 @@ def storages_add
   path = stdin_read_val('Path')
   url = stdin_read_val('Url')
   size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
+  copy_id = stdin_read_val('Copy id').to_i
   begin
-    storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit)
+    path = path +'/' unless path[-1] == '/'
+    path = '/' + path unless path[0] == '/'
+    storage = FC::Storage.new(:name => name, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_id => copy_id)
     print "Calc current size.. "
     size = storage.file_size('')
     puts "ok"
@@ -54,7 +54,8 @@ def storages_add
   Path:       #{path} 
   Url:        #{url}
   Size:       #{size_to_human size}
-  Size limit: #{size_to_human size_limit}}
+  Size limit: #{size_to_human size_limit}
+  Copy id:    #{copy_id}}
   s = Readline.readline("Continue? (y/n) ", false).strip.downcase
   puts ""
   if s == "y" || s == "yes"
@@ -72,11 +73,7 @@ def storages_add
 end
 
 def storages_rm
-  name = ARGV[2]
-  storage = FC::Storage.where('name = ?', name).first
-  if !storage
-    puts "Storage #{name} not found."
-  else
+  if storage = find_storage
     s = Readline.readline("Continue? (y/n) ", false).strip.downcase
     puts ""
     if s == "y" || s == "yes"
@@ -87,3 +84,73 @@ def storages_rm
     end
   end
 end
+
+def storages_update_size
+  if storage = find_storage
+    print "Calc current size.. "
+    size = storage.file_size('')
+    storage.size = size
+    begin
+      storage.save
+    rescue Exception => e
+      puts "Error: #{e.message}"
+      exit
+    end
+    puts "ok"
+  end
+end
+
+def storages_change
+  if storage = find_storage
+    puts "Change storage #{storage.name}"
+    host = stdin_read_val("Host (now #{storage.host})", true)
+    path = stdin_read_val("Path (now #{storage.path})", true)
+    url = stdin_read_val("Url (now #{storage.url})", true)
+    size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
+    copy_id = stdin_read_val("Copy id (now #{storage.copy_id})", true)
+    
+    storage.host = host unless host.empty?
+    if !path.empty? && path != storage.path
+      path = path +'/' unless path[-1] == '/'
+      path = '/' + path unless path[0] == '/'
+      storage.path = path 
+      print "Calc current size.. "
+      storage.size = storage.file_size('')
+      puts "ok"
+    end
+    storage.url = url unless url.empty?
+    storage.size_limit = human_to_size(size_limit) unless size_limit.empty?
+    storage.copy_id = copy_id.to_i
+    
+    puts %Q{\nStorage
+    Name:       #{storage.name}
+    Host:       #{storage.host} 
+    Path:       #{storage.path} 
+    Url:        #{storage.url}
+    Size:       #{size_to_human storage.size}
+    Size limit: #{size_to_human storage.size_limit}
+    Copy id:    #{copy_id}}
+    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+    puts ""
+    if s == "y" || s == "yes"
+      begin
+        storage.save
+      rescue Exception => e
+        puts "Error: #{e.message}"
+        exit
+      end
+      puts "ok"
+    else
+      puts "Canceled."
+    end
+  end
+end
+
+private
+
+def find_storage
+  name = ARGV[2]
+  storage = FC::Storage.where('name = ?', name).first
+  puts "Storage #{name} not found." if !storage
+  storage
+end

data/lib/utils.rb

@@ -40,9 +40,9 @@ def human_to_size(size)
   result.to_i
 end
 
-def stdin_read_val(name)
+def stdin_read_val(name, can_empty = false)
   while val = Readline.readline("#{name}: ", false).strip.downcase
-    if val.empty?
+    if val.empty? && !can_empty
       puts "Input non empty #{name}."
     else 
       if block_given?

data/test/daemon_test.rb

@@ -37,12 +37,12 @@ class DaemonTest < Test::Unit::TestCase
       `dd if=/dev/urandom of=#{@@test_file_path} bs=1M count=1 2>&1`
       
       @@storages = []
-      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :size_limit => 1000000000)
-      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :size_limit => 1000000000)
-      @@storages << FC::Storage.new(:name => 'host1-sdc', :host => 'host1', :path => '/tmp/host1-sdc/', :size_limit => 1000000000)
+      @@storages << FC::Storage.new(:name => 'host1-sda', :host => 'host1', :path => '/tmp/host1-sda/', :copy_id => 1, :size_limit => 1000000000)
+      @@storages << FC::Storage.new(:name => 'host1-sdb', :host => 'host1', :path => '/tmp/host1-sdb/', :copy_id => 2, :size_limit => 1000000000)
+      @@storages << FC::Storage.new(:name => 'host1-sdc', :host => 'host1', :path => '/tmp/host1-sdc/', :copy_id => 1, :size_limit => 1000000000)
       @@storages.each { |storage| storage.save}
       
-      @@policy = FC::Policy.new(:storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2)
+      @@policy = FC::Policy.new(:create_storages => 'host1-sda,host1-sdb,host1-sdc', :copy_storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2, :name => 'policy 1')
       @@policy.save
       
       # wait for running fc-daemon
@@ -70,9 +70,9 @@ class DaemonTest < Test::Unit::TestCase
     
     FC::Storage.any_instance.stubs(:host).returns('host1')
     FC::Storage.stubs(:curr_host).returns('host1')
-    assert_nothing_raised { @item1 = FC::Item.create_from_local(@@test_file_path, 'test1', @@policy, {:tag => 'test1'}) }
-    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'test2', @@policy, {:tag => 'test2'}) }
-    assert_nothing_raised { @item3 = FC::Item.create_from_local(@@test_file_path, 'test3', @@policy, {:tag => 'test3'}) }
+    assert_nothing_raised { @item1 = FC::Item.create_from_local(@@test_file_path, 'bla/bla/test1', @@policy, {:tag => 'test1'}) }
+    assert_nothing_raised { @item2 = FC::Item.create_from_local(@@test_file_path, 'bla/bla/test2', @@policy, {:tag => 'test2'}) }
+    assert_nothing_raised { @item3 = FC::Item.create_from_local(@@test_file_path, 'bla/bla/test3', @@policy, {:tag => 'test3'}) }
     
     @@policy.copies = 3
     @@policy.save
@@ -81,7 +81,7 @@ class DaemonTest < Test::Unit::TestCase
     # wait for copy
     [1, 2, 3].each do |i|
       ['b', 'c'].each do |j|
-        assert_equal `du -b /tmp/host1-sda/test$i 2>&1`.to_i, `du -b /tmp/host$i-sd$j/test$i 2>&1`.to_i
+        assert_equal `du -b /tmp/host1-sda/bla/bla/test$i 2>&1`.to_i, `du -b /tmp/host$i-sd$j/bla/bla/test$i 2>&1`.to_i
       end
     end
         
@@ -91,7 +91,7 @@ class DaemonTest < Test::Unit::TestCase
     item_storage.status = 'delete'
     item_storage.save
     sleep 2
-    assert_equal 0, `du -b /tmp/host1-sdc/test1 2>&1`.to_i
+    assert_equal 0, `du -b /tmp/host1-sdc/bla/bla/test1 2>&1`.to_i
     
     assert_equal @@errors_count, FC::Error.where.count, "new errors in errors table"
   end

data/test/db_test.rb

@@ -15,9 +15,9 @@ class DbTest < Test::Unit::TestCase
       @@storages_ids = storages.map{|storage| storage.save; storage.id }
       
       policies = []
-      policies << FC::Policy.new(:storages => 'rec1-sda,rec1-sdd', :copies => 2)
-      policies << FC::Policy.new(:storages => 'rec1-sda,bla,rec2-sdd', :copies => 3)
-      policies << FC::Policy.new(:storages => 'bla,rec1-sda,test', :copies => 4)
+      policies << FC::Policy.new(:create_storages => 'rec1-sda,rec1-sdd', :copy_storages => 'rec2-sda,rec2-sdd', :copies => 2, :name => 'policy 1')
+      policies << FC::Policy.new(:create_storages => 'rec1-sda,bla,rec2-sdd', :copy_storages => 'rec2-sda,bla,rec1-sdd', :copies => 3, :name => 'policy 2')
+      policies << FC::Policy.new(:create_storages => 'bla,rec1-sda,test', :copy_storages => 'bla,rec2-sda,test', :copies => 4, :name => 'policy 3')
       @@policies_ids = policies.map{|policy| policy.save; policy.id }
       
       items = []
@@ -75,26 +75,39 @@ class DbTest < Test::Unit::TestCase
   
   should "policies and storages" do
     assert @policies.count > 0, 'Policies not loaded'
-    assert_equal 'rec1-sda,rec1-sdd', @policies[0].storages, "Policy (id=#{@policies[0].id}) incorrect storages"
-    assert_equal 'rec1-sda,rec2-sdd', @policies[1].storages, "Policy (id=#{@policies[0].id}) incorrect storages"
-    assert_equal 'rec1-sda', @policies[2].storages, "Policy (id=#{@policies[0].id}) incorrect storages"
-    assert_raise(Mysql2::Error, 'Create policy with incorrect storages') { FC::Policy.new(:storages => 'bla,test').save }
+    assert_equal 'rec1-sda,rec1-sdd', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
+    assert_equal 'rec1-sda,rec2-sdd', @policies[1].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
+    assert_equal 'rec1-sda', @policies[2].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages"
+    assert_equal 'rec2-sda,rec2-sdd', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
+    assert_equal 'rec2-sda,rec1-sdd', @policies[1].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
+    assert_equal 'rec2-sda', @policies[2].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages"
+    
+    assert_raise(Mysql2::Error, 'Create policy with uniq name') { FC::Policy.new(:create_storages => 'bla,test', :copy_storages => 'rec2-sda', :name => 'policy 1').save }
+    assert_raise(Mysql2::Error, 'Create policy with incorrect create_storages') { FC::Policy.new(:create_storages => 'bla,test', :copy_storages => 'rec2-sda', :name => 'new policy').save }
+    assert_raise(Mysql2::Error, 'Create policy with incorrect copy_storages') { FC::Policy.new(:create_storages => 'rec1-sda', :copy_storages => 'bla,test', :name => 'new policy').save }
+    
     assert_raise(Mysql2::Error, 'Change storage name with linked polices') { @storages[0].name = 'blabla'; @storages[0].save }
     assert_raise(Mysql2::Error, 'Delete storage name with linked polices') { @storages[0].delete }
     assert_nothing_raised { @storages[6].name = 'rec2-sdc-new'; @storages[6].save }
     @storages[3].name = 'rec1-sdd-new' #rec1-sdd
     @storages[3].save
     @policies[0].reload
-    assert_equal 'rec1-sda', @policies[0].storages, "Policy (id=#{@policies[0].id}) incorrect storages after storage change"
+    @policies[1].reload
+    assert_equal 'rec1-sda', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages after storage change"
+    assert_equal 'rec2-sda', @policies[1].copy_storages, "Policy (id=#{@policies[1].id}) incorrect copy_storages after storage change"
     @storages[7].delete  #rec2-sdd
     @@storages_ids.delete(@storages[7].id)
+    @policies[0].reload
     @policies[1].reload
-    assert_equal 'rec1-sda', @policies[1].storages, "Policy (id=#{@policies[1].id}) incorrect storages after storage delete"
-    @policies[0].storages = 'rec1-sda,rec2-sda,bla bla'
+    assert_equal 'rec1-sda', @policies[1].create_storages, "Policy (id=#{@policies[1].id}) incorrect create_storages after storage delete"
+    assert_equal 'rec2-sda', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages after storage delete"
+    @policies[0].create_storages = 'rec2-sda,rec1-sda,bla bla'
+    @policies[0].copy_storages = 'rec1-sdb,rec2-sda,bla bla'
     @policies[0].save
     @policies[0].reload
-    assert_equal 'rec1-sda,rec2-sda', @policies[0].storages, "Policy (id=#{@policies[0].id}) incorrect storages after change"
-    assert_raise(Mysql2::Error, 'Save empty policy storage') { @policies[0].storages = 'blabla'; @policies[0].save }
+    assert_equal 'rec2-sda,rec1-sda', @policies[0].create_storages, "Policy (id=#{@policies[0].id}) incorrect create_storages after change"
+    assert_equal 'rec1-sdb,rec2-sda', @policies[0].copy_storages, "Policy (id=#{@policies[0].id}) incorrect copy_storages after change"
+    assert_raise(Mysql2::Error, 'Save empty policy storage') { @policies[0].create_storages = 'blabla'; @policies[0].save }
   end
   
   should "item_storages doubles" do

data/test/functional_test.rb

@@ -20,10 +20,10 @@ class FunctionalTest < Test::Unit::TestCase
       @@storages.each { |storage| storage.save}
       
       @@policies = []
-      @@policies << FC::Policy.new(:storages => 'host1-sda,host2-sda', :copies => 2)
-      @@policies << FC::Policy.new(:storages => 'host1-sdb,host2-sdb', :copies => 2)
-      @@policies << FC::Policy.new(:storages => 'host3-sda', :copies => 1)
-      @@policies << FC::Policy.new(:storages => 'host2-sda', :copies => 1)
+      @@policies << FC::Policy.new(:create_storages => 'host1-sda,host2-sda', :copy_storages => 'host1-sdb', :copies => 2, :name => 'policy 1')
+      @@policies << FC::Policy.new(:create_storages => 'host1-sdb,host2-sdb', :copy_storages => 'host1-sdb', :copies => 2, :name => 'policy 2')
+      @@policies << FC::Policy.new(:create_storages => 'host3-sda', :copy_storages => 'host1-sdb', :copies => 1, :name => 'policy 3')
+      @@policies << FC::Policy.new(:create_storages => 'host2-sda', :copy_storages => 'host1-sdb', :copies => 1, :name => 'policy 4')
       @@policies.each { |policy| policy.save}
     end
     def shutdown
@@ -41,10 +41,10 @@ class FunctionalTest < Test::Unit::TestCase
   end
   
   should "item create_from_local successful" do
-    assert_nothing_raised { @item = FC::Item.create_from_local(@@test_file_path, 'test1', @@policies[0], {:tag => 'test'}) }
+    assert_nothing_raised { @item = FC::Item.create_from_local(@@test_file_path, '/bla/bla/test1', @@policies[0], {:tag => 'test'}) }
     assert_kind_of FC::Item, @item
-    assert_equal `du -b /tmp/host1-sda/test1 2>&1`.to_i, `du -b #{@@test_file_path} 2>&1`.to_i
-    assert_equal `du -b /tmp/host1-sda/test1 2>&1`.to_i, @item.size
+    assert_equal `du -b /tmp/host1-sda/bla/bla/test1 2>&1`.to_i, `du -b #{@@test_file_path} 2>&1`.to_i
+    assert_equal `du -b /tmp/host1-sda/bla/bla/test1 2>&1`.to_i, @item.size
     assert_equal 'ready', @item.status
     item_storages = @item.get_item_storages
     assert_equal 1, item_storages.count
@@ -53,12 +53,6 @@ class FunctionalTest < Test::Unit::TestCase
     assert_equal 'host1-sda', item_storage.storage_name
   end
   
-  should "item create_from_local error local path" do
-    errors_count = FC::Error.where.count
-    assert_raise(RuntimeError) { FC::Item.create_from_local(@@test_file_path, 'test2', @@policies[1], {:tag => 'test'}) }
-    assert_equal errors_count+1, FC::Error.where.count, "Error not saved after error local path"
-  end
-  
   should "item create_from_local replace" do
     @item  = FC::Item.new(:name => 'test2', :policy_id => @@policies[0].id)
     @item.save

data/test/policy_test.rb

@@ -4,11 +4,13 @@ class PolicyTest < Test::Unit::TestCase
   class << self
     def startup
       @@storages = []
-      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :size_limit => 10)
-      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :copy_id => 1, :size_limit => 10)
+      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_id => 2 , :size_limit => 100)
       @@storages.each {|storage| storage.save}
-      
-      @@policy = FC::Policy.new(:storages => 'rec1-sda,rec2-sda', :copies => 1)
+      @@storage3 = FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_id => 3 , :size_limit => 10)
+      @@storage3.save
+            
+      @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda', :copy_storages => 'rec1-sda,rec2-sda', :copies => 1, :name => 'policy 1')
       @@policy.save
     end
     def shutdown
@@ -17,26 +19,63 @@ class PolicyTest < Test::Unit::TestCase
     end
   end 
 
-  should "get_storages" do
+  should "get_create_storages" do
     FC::Policy.storages_cache_time = 10
-    assert_same_elements @@storages.map(&:id), @@policy.get_storages.map(&:id)
-    FC::Storage.new(:name => 'rec2-sdc', :host => 'rec2').save
-    @@policy.storages = 'rec1-sda,rec2-sda,rec2-sdc'
+    assert_same_elements @@storages.map(&:id), @@policy.get_create_storages.map(&:id)
+    @@policy.create_storages = 'rec1-sda,rec2-sda,rec3-sda'
     @@policy.save
-    assert_equal @@storages.size, @@policy.get_storages.size
+    assert_equal @@storages.size, @@policy.get_create_storages.size
     FC::Policy.storages_cache_time = 0
-    assert_equal @@storages.size+1, @@policy.get_storages.size
+    assert_equal @@storages.size+1, @@policy.get_create_storages.size
   end
   
-  should "get_proper_storage" do
+  should "get_copy_storages" do
+    @@policy.copy_storages = 'rec1-sda,rec2-sda'
+    @@policy.save
+    FC::Policy.storages_cache_time = 10
+    assert_same_elements @@storages.map(&:id), @@policy.get_copy_storages.map(&:id)
+    @@policy.copy_storages = 'rec1-sda,rec2-sda,rec3-sda'
+    @@policy.save
+    assert_equal @@storages.size, @@policy.get_copy_storages.size
     FC::Policy.storages_cache_time = 0
-    assert_nil @@policy.get_proper_storage(1), 'all storages down'
+    assert_equal @@storages.size+1, @@policy.get_copy_storages.size
+  end
+  
+  should "get_proper_storage_for_create" do
+    @@storages.each {|storage| storage.check_time = 0; storage.save}
+    @@storage3.check_time = 0
+    @@storage3.save
+    FC::Policy.storages_cache_time = 0
+    assert_nil @@policy.get_proper_storage_for_create(1), 'all storages down'
     @@storages[0].update_check_time
-    assert_equal @@storages[0].id, @@policy.get_proper_storage(1).id, 'first storages up'
-    assert_nil @@policy.get_proper_storage(20), 'first storage full'
+    assert_equal @@storages[0].id, @@policy.get_proper_storage_for_create(5).id, 'first storages up'
+    assert_nil @@policy.get_proper_storage_for_create(20), 'first storage full'
     @@storages[1].update_check_time
-    assert_equal @@storages[1].id, @@policy.get_proper_storage(20).id, 'second storages up'
-    assert_nil @@policy.get_proper_storage(1000), 'all storages full'
+    assert_equal @@storages[1].id, @@policy.get_proper_storage_for_create(20).id, 'second storages up'
+    assert_nil @@policy.get_proper_storage_for_create(1000), 'all storages full'
   end
   
+  should "get_proper_storage_for_copy" do
+    @@storages.each {|storage| storage.check_time = 0; storage.save}
+    FC::Policy.storages_cache_time = 0
+    assert_nil @@policy.get_proper_storage_for_copy(1), 'all storages down'
+    @@storages[0].update_check_time
+    @@storage3.update_check_time
+    assert_equal @@storages[0].id, @@policy.get_proper_storage_for_copy(5).id, 'first storages up'
+    assert_nil @@policy.get_proper_storage_for_copy(20), 'first storage full'
+    @@storages[1].update_check_time
+    assert_equal @@storages[1].id, @@policy.get_proper_storage_for_copy(20).id, 'second storages up'
+    assert_nil @@policy.get_proper_storage_for_copy(1000), 'all storages full'
+    
+    @@policy.copy_storages = 'rec3-sda,rec1-sda,rec2-sda'
+    @@policy.save
+    assert_equal 'rec3-sda', @@policy.get_proper_storage_for_copy(1).name, 'first storage in copy_storages'
+    assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 2).name, 'storage by copy_id'
+    @@policy.copy_storages = 'rec1-sda,rec3-sda'
+    @@policy.save
+    assert_equal 'rec3-sda', @@policy.get_proper_storage_for_copy(1, 2).name, 'storage by copy_id'
+    @@policy.copy_storages = 'rec2-sda,rec3-sda,rec1-sda'
+    @@policy.save
+    assert_equal 'rec2-sda', @@policy.get_proper_storage_for_copy(1, 4).name, 'storage by copy_id'
+  end
 end

data/test/storage_test.rb

@@ -7,7 +7,6 @@ class StorageTest < Test::Unit::TestCase
   
   should "initialize" do
     assert_raise(RuntimeError) { FC::Storage.new :path => 'test' }
-    assert_raise(RuntimeError) { FC::Storage.new :path => '/test' }
     assert_raise(RuntimeError) { FC::Storage.new :path => 'test/' }
     assert_nothing_raised { FC::Storage.new :path => '/test/' }
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-13 00:00:00.000000000 Z
+date: 2013-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -186,7 +186,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3979001515347744911
+      hash: -919880063
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -195,7 +195,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3979001515347744911
+      hash: -919880063
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24