filecluster 0.3.7 → 0.3.8

data/README.md

@@ -5,6 +5,7 @@ A set of scripts to manage files on multiple dc/host/drive.
 Item (storage unit)- file or folder.
 Storage - folder, usually separate drive.
 Policy - rule for selecting storage to store the item and сreate copies.
+Copy rule - additional rule for copies сreate.
 
 Daemon monitors the availability of each storage and copy files between them.
 
@@ -31,7 +32,20 @@ Selecting available storage to store item by policy.create_storages (from left t
     
 ## Copy policy
 
-Selecting available storage to copy item by storage.copy_storages (from left to right).
+Used first available storage.
+First try copy rules (copy_storages field, from left to right).
+After that storage.copy_storages (from left to right).
+
+Copy rule is ruby expression.
+Can be used the following variables:
+
+* item_id
+* size
+* item_copies
+* name
+* tag
+* dir
+* src_storage (FC::Storage instance)
 
 ## Variables
 

data/bin/fc-manage

@@ -46,6 +46,16 @@ Command:
    errors [<count>]    show last count (default 10) errors
    host_info [<host>]  show info for host (default current host)
    items_info          show items statistics       
+}],
+  'copy_rules' => [
+    'show and manage copy rules', 
+    %q{Usage: fc-manage [options] copy_rules <command>
+Command:
+   list                show all copy rules
+   show <id>           show full info for copy rule   
+   add                 add new copy rule
+   rm <id>             delete copy rule
+   change <id>         change copy rules attributes       
 }],
   'var' => [
     'show and change FC::Var', 

data/lib/daemon/global_daemon_thread.rb

@@ -33,7 +33,7 @@ class GlobalDaemonThread < BaseThread
     all_policies.each do |policy|
       next if policy.copies.to_i < 2
       copies = (1..policy.copies.to_i-1).to_a.join(',')
-      sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, GROUP_CONCAT(ist.storage_name ORDER BY ist.id) as storages "+
+      sql = "SELECT i.id as item_id, i.size, i.copies as item_copies, i.name, i.tag, i.dir, GROUP_CONCAT(ist.storage_name ORDER BY ist.id) as storages "+
         "FROM #{FC::Item.table_name} as i, #{FC::ItemStorage.table_name} as ist WHERE i.policy_id = #{policy.id} AND "+
         "ist.item_id = i.id AND i.copies IN (#{copies}) AND i.status = 'ready' AND ist.status <> 'delete' GROUP BY i.id LIMIT #{limit}"
       r = FC::DB.query(sql)
@@ -46,8 +46,23 @@ class GlobalDaemonThread < BaseThread
           $log.warn("GlobalDaemonThread: ItemStorage count >= policy.copies for item #{row['item_id']}")
         else
           src_storage = all_storages.detect{|s| item_storages.first == s.name}
-          storage = src_storage.get_proper_storage_for_copy(row['size'], item_storages) if src_storage 
+          if src_storage
+            storage = FC::CopyRule.get_proper_storage_for_copy(
+              :item_id     => row['item_id'],
+              :size        => row['size'],
+              :item_copies => row['item_copies'],
+              :name        => row['name'],
+              :tag         => row['tag'],
+              :dir         => row['dir'].to_i == 1,
+              :src_storage => src_storage,
+              :exclude     => item_storages
+            )
+            storage = src_storage.get_proper_storage_for_copy(row['size'], item_storages) unless storage
+          end
           if storage
+            puts row
+            puts src_storage.name+'>'
+            puts storage.name
             FC::Item.new(:id => row['item_id']).make_item_storage(storage, 'copy')
           else
             error 'No available storage', :item_id => row['item_id']

data/lib/fc/copy_rule.rb

@@ -0,0 +1,79 @@
+# encoding: utf-8
+
+module FC
+  class CopyRule < DbBase
+    set_table :copy_rules, 'rule, copy_storages'
+    
+    class << self
+      attr_accessor :rules_cache_time, :get_rules_mutex, :copy_storages_cache_time, :get_copy_storages_mutex
+    end
+    @rules_cache_time = 20 # ttl for rules cache
+    @copy_storages_cache_time = 20 # ttl for rule copy_storages
+    @get_rules_mutex = Mutex.new
+    @get_copy_storages_mutex = Mutex.new
+    
+    def self.all
+      get_rules_mutex.synchronize do
+        unless @all_rules_cache && Time.new.to_i - @get_all_rules_time.to_i < rules_cache_time
+          @get_all_rules_time = Time.new.to_i
+          @all_rules_cache = where("1")
+        end
+      end
+      @all_rules_cache
+    end
+    
+    def self.check_all(item_id, size, item_copies, name, tag, dir, src_storage)
+      all.select do |r|
+        r.check(item_id, size, item_copies, name, tag, dir, src_storage)
+      end
+    end
+    
+    # get available storage for copy
+    def self.get_proper_storage_for_copy(options)
+      exclude = options[:exclude] || []
+      rules = check_all(options[:item_id].to_i, options[:size].to_i, options[:item_copies].to_i, options[:name].to_s, options[:tag].to_s, options[:dir] ? true : false, options[:src_storage])
+      result = nil
+      rules.each do |rule|
+        result = rule.get_copy_storages.select do |storage|
+          !exclude.include?(storage.name) && storage.up? && storage.size + options[:size].to_i < storage.size_limit
+        end.first
+        break if result
+      end
+      result
+    end
+    
+    def get_copy_storages
+      self.class.get_copy_storages_mutex.synchronize do
+        unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.copy_storages_cache_time
+          @get_copy_storages_time = Time.new.to_i
+          names = copy_storages.to_s.split(',').map{|s| "'#{s}'"}.join(',')
+          @copy_storages_cache = names.empty? ? [] : FC::Storage.where("name IN (#{names}) ORDER BY FIELD(name, #{names})")
+        end
+      end
+      @copy_storages_cache
+    end
+    
+    def check(item_id, size, item_copies, name, tag, dir, src_storage)
+      return false unless rule
+      $SELF = 4
+      r = eval(rule)
+      $SELF = 0
+      r ? true : false
+    end
+    
+    def test
+      storage = FC::Storage.new(
+        :id => 1,
+        :name => 'test_storage',
+        :host => 'test_host',
+        :path => '/bla/bla',
+        :url  => 'http://bla',
+        :size => 1000,
+        :size_limit => 9999,
+        :check_time => Time.new.to_i, 
+        :copy_storages => 'a,b,c'
+      )
+      check(3, 1, 1, 'test/item', 'tag', false, storage)
+    end
+  end
+end

data/lib/fc/db.rb

@@ -179,6 +179,15 @@ module FC
       })
       FC::DB.connect.query("CREATE TRIGGER fc_errors_before_insert BEFORE INSERT on #{@prefix}errors FOR EACH ROW BEGIN #{proc_time} END")
       
+      FC::DB.connect.query(%{
+        CREATE TABLE #{@prefix}copy_rules (
+          id int NOT NULL AUTO_INCREMENT,
+          copy_storages text NOT NULL DEFAULT '',
+          rule text DEFAULT NULL,
+          PRIMARY KEY (id)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci
+      })
+      
       FC::DB.connect.query(%{
         CREATE TABLE #{@prefix}vars (
           name varchar(255) DEFAULT NULL,

data/lib/fc/policy.rb

@@ -28,11 +28,16 @@ module FC
       @create_storages_cache
     end
     
-    # get available storage for create by size
-    def get_proper_storage_for_create(size, exclude = [])
-      get_create_storages.detect do |storage|
+    # get available storages for create by size
+    def get_proper_storages_for_create(size, exclude = [])
+      get_create_storages.select do |storage|
         !exclude.include?(storage.name) && storage.up? && storage.size + size < storage.size_limit
       end
     end
+    
+    # get available storage for create by size
+    def get_proper_storage_for_create(size, exclude = [])
+      get_proper_storages_for_create(size, exclude).first
+    end
   end
 end

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = "0.3.7"
+  VERSION = "0.3.8"
 end

data/lib/filecluster.rb

@@ -6,6 +6,7 @@ require "fc/item_storage"
 require "fc/item"
 require "fc/policy"
 require "fc/storage"
+require "fc/copy_rule"
 require "fc/error"
 require "fc/var"
 

data/lib/manage.rb

@@ -2,4 +2,5 @@ require 'readline'
 require "manage/policies"
 require "manage/storages"
 require "manage/show"
+require "manage/copy_rules"
 require "manage/var"

data/lib/manage/copy_rules.rb

@@ -0,0 +1,107 @@
+# encoding: utf-8
+require 'shellwords'
+
+def copy_rules_list
+  rules = FC::CopyRule.where("1 ORDER BY id")
+  if rules.size == 0
+    puts "No rules."
+  else
+    rules.each do |rule|
+      puts "##{rule.id}, copy storages: #{rule.copy_storages}, rule: #{rule.rule}"
+    end
+  end
+end
+
+def copy_rules_show
+  if rule = find_rule
+    #count = FC::DB.query("SELECT count(*) as cnt FROM #{FC::ItemStorage.table_name} WHERE storage_name='#{Mysql2::Client.escape(storage.name)}'").first['cnt']
+    puts %Q{Rule
+  Id:             #{rule.id}
+  Copy storages:  #{rule.copy_storages}
+  Rule:           #{rule.rule}}
+  end
+end
+
+def copy_rules_add
+  puts "Add copy rule"
+  copy_storages = stdin_read_val('Copy storages')
+  storages = FC::Storage.where.map(&:name)
+  copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
+  rule_str = stdin_read_val('Rule')
+  
+  begin
+    rule = FC::CopyRule.new(:rule => rule_str, :copy_storages => copy_storages)
+    rule.test
+  rescue Exception => e
+    puts "Error: #{e.message}"
+    exit
+  end
+  puts %Q{\nRule
+  Copy storages:  #{rule.copy_storages}
+  Rule:           #{rule.rule}}
+  s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+  puts ""
+  if s == "y" || s == "yes"
+    begin
+      rule.save
+    rescue Exception => e
+      puts "Error: #{e.message}"
+      exit
+    end
+    puts "ok"
+  else
+    puts "Canceled."
+  end
+end
+
+def copy_rules_rm
+  if rule = find_rule
+    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+    puts ""
+    if s == "y" || s == "yes"
+      rule.delete
+      puts "ok"
+    else
+      puts "Canceled."
+    end
+  end
+end
+
+def copy_rules_change
+  if rule = find_rule
+    puts "Change rule #{rule.id}"
+    copy_storages = stdin_read_val("Copy storages (now #{rule.copy_storages})", true)
+    storages = FC::Storage.where.map(&:name)
+    rule.copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip unless copy_storages.empty?
+    rule_str = stdin_read_val("Rule (now #{rule.rule})", true)
+    rule.rule = rule_str unless rule_str.empty?
+    
+    puts %Q{\nRule
+    Id:             #{rule.id}
+    Copy storages:  #{rule.copy_storages}
+    Rule:           #{rule.rule}}
+    s = Readline.readline("Continue? (y/n) ", false).strip.downcase
+    puts ""
+    if s == "y" || s == "yes"
+      begin
+        rule.test
+        rule.save
+      rescue Exception => e
+        puts "Error: #{e.message}"
+        exit
+      end
+      puts "ok"
+    else
+      puts "Canceled."
+    end
+  end
+end
+
+private
+
+def find_rule
+  id = ARGV[2]
+  rule = FC::CopyRule.where('id = ?', id).first
+  puts "Rule #{id} not found." unless rule
+  rule
+end

data/test/copy_rule_test.rb

@@ -0,0 +1,80 @@
+require 'helper'
+
+class CopyRuleTest < Test::Unit::TestCase
+  class << self
+    def startup
+      @@rules = []
+      @@rules << FC::CopyRule.new(:copy_storages => 'rec2-sda,rec3-sda', :rule => 'size < 100')
+      @@rules << FC::CopyRule.new(:copy_storages => 'rec1-sda,rec3-sda', :rule => 'name.match(/^test1/)')
+      @@rules.each {|rule| rule.save}
+      
+      @@storages = []
+      @@storages << FC::Storage.new(:name => 'rec1-sda', :host => 'rec1', :size => 0, :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :size_limit => 10)
+      @@storages << FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 0, :size_limit => 100)
+      @@storages.each {|storage| storage.save}
+    end
+    def shutdown
+      FC::DB.query("DELETE FROM copy_rules")
+      FC::DB.query("DELETE FROM storages")
+    end
+  end
+  
+  should "rule check" do
+    assert_equal true, @@rules[0].test
+    assert_equal true, @@rules[0].check(1, 1, 1, '', '', false, nil)
+    assert_equal false, @@rules[0].check(1, 100, 1, '', '', false, nil)
+    assert_equal false, @@rules[1].test
+    assert_equal false, @@rules[1].check(1, 1, 1, '', '', false, nil)
+    assert_equal true, @@rules[1].check(1, 1, 1, 'test1/test', '', false, nil)
+  end
+  
+  should "get_copy_storages" do
+    FC::CopyRule.copy_storages_cache_time = 10
+    assert_equal 'rec2-sda,rec3-sda', @@rules[0].get_copy_storages.map(&:name).join(',')
+    @@rules[0].copy_storages = 'rec3-sda,rec2-sda'
+    @@rules[0].save
+    assert_equal 'rec2-sda,rec3-sda', @@rules[0].get_copy_storages.map(&:name).join(',')
+    FC::CopyRule.copy_storages_cache_time = 0
+    assert_equal 'rec3-sda,rec2-sda', @@rules[0].get_copy_storages.map(&:name).join(',')
+  end
+  
+  should "all rules" do
+    FC::CopyRule.rules_cache_time = 10
+    assert_equal @@rules.map(&:id), FC::CopyRule.all.map(&:id)
+    assert_equal 'rec1-sda,rec3-sda', FC::CopyRule.all[1].copy_storages
+    @@rules[1].copy_storages = 'rec1-sda,rec2-sda'
+    @@rules[1].save
+    assert_equal 'rec1-sda,rec3-sda', FC::CopyRule.all[1].copy_storages
+    FC::CopyRule.rules_cache_time = 0
+    assert_equal 'rec1-sda,rec2-sda', FC::CopyRule.all[1].copy_storages
+  end
+  
+  should "check_all" do
+    assert_equal [], FC::CopyRule.check_all(1, 100, 1, '', '', false, nil)
+    assert_same_elements [@@rules[0].id], FC::CopyRule.check_all(1, 1, 1, '', '', false, nil).map(&:id) 
+    assert_same_elements [@@rules[0].id, @@rules[1].id], FC::CopyRule.check_all(1, 1, 1, 'test1/test', '', false, nil).map(&:id)
+  end
+  
+  should "get_proper_storage_for_copy" do
+    @@rules[0].copy_storages = 'rec2-sda,rec3-sda'
+    @@rules[1].copy_storages = 'rec1-sda,rec3-sda'
+    @@rules[0].save
+    @@rules[1].save
+    FC::CopyRule.rules_cache_time = 0
+    FC::CopyRule.copy_storages_cache_time = 0
+    
+    @@storages.each {|storage| storage.check_time = 0; storage.save}
+    assert_nil FC::CopyRule.get_proper_storage_for_copy(:size => 1), 'all storages down'
+
+    @@storages[0].update_check_time
+    assert_nil FC::CopyRule.get_proper_storage_for_copy(:size => 1), 'first storage up, but no rules with storage'
+    assert_equal 'rec1-sda', FC::CopyRule.get_proper_storage_for_copy(:name => 'test1/test').name, 'first storage up'
+    assert_nil FC::CopyRule.get_proper_storage_for_copy(:name => 'test1/test', :size => 200), 'first storage full'
+    assert_nil FC::CopyRule.get_proper_storage_for_copy(:name => 'test1/test', :exclude => ['rec1-sda']), 'exclude'
+    @@storages[1].update_check_time
+    assert_equal 'rec2-sda', FC::CopyRule.get_proper_storage_for_copy(:name => 'test1/test', :size => 5).name, 'second storages up, small file'
+    assert_equal 'rec1-sda', FC::CopyRule.get_proper_storage_for_copy(:name => 'test1/test', :size => 20).name, 'second storages up, big file'
+    assert_nil FC::CopyRule.get_proper_storage_for_copy(:name => 'test1/test', :size => 1000), 'second storages up, big file'
+  end
+end

data/test/daemon_test.rb

@@ -53,6 +53,9 @@ class DaemonTest < Test::Unit::TestCase
       @@policy = FC::Policy.new(:create_storages => 'host1-sda,host1-sdb,host1-sdc', :copies => 2, :name => 'policy 1')
       @@policy.save
       
+      @@rule = FC::CopyRule.new(:copy_storages => 'host1-sdc', :rule => 'name == "bla/bla/test3"')
+      @@rule.save
+      
       # wait for running fc-daemon
       Timeout::timeout(5) do
         while @stotage_checks < @@storages.size
@@ -77,6 +80,7 @@ class DaemonTest < Test::Unit::TestCase
     @@storages.each {|storage| storage.reload}
     assert @@storages[0].up?, "Storage #{@@storages[0].name} down" 
     assert @@storages[1].up?, "Storage #{@@storages[1].name} down"
+    assert @@storages[2].up?, "Storage #{@@storages[2].name} down"
     
     FC::Storage.any_instance.stubs(:host).returns('host1')
     FC::Storage.stubs(:curr_host).returns('host1')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-25 00:00:00.000000000 Z
+date: 2014-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb-readline
@@ -152,6 +152,7 @@ files:
 - lib/daemon/run_tasks_thread.rb
 - lib/daemon/update_tasks_thread.rb
 - lib/fc/base.rb
+- lib/fc/copy_rule.rb
 - lib/fc/db.rb
 - lib/fc/error.rb
 - lib/fc/item.rb
@@ -162,12 +163,14 @@ files:
 - lib/fc/version.rb
 - lib/filecluster.rb
 - lib/manage.rb
+- lib/manage/copy_rules.rb
 - lib/manage/policies.rb
 - lib/manage/show.rb
 - lib/manage/storages.rb
 - lib/manage/var.rb
 - lib/utils.rb
 - test/base_test.rb
+- test/copy_rule_test.rb
 - test/daemon_test.rb
 - test/db_test.rb
 - test/error_test.rb
@@ -193,7 +196,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -247520372150803461
+      hash: -1801267482625081315
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -202,7 +205,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -247520372150803461
+      hash: -1801267482625081315
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -211,6 +214,7 @@ specification_version: 3
 summary: Distributed storage
 test_files:
 - test/base_test.rb
+- test/copy_rule_test.rb
 - test/daemon_test.rb
 - test/db_test.rb
 - test/error_test.rb