filecluster 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d7f682fa6213abbdb15d573d39edcf21b458ec51
-  data.tar.gz: a193f594ba3e08ec6442ee9b048bffd42174c162
+  metadata.gz: 09926dfab03ca82fe41a029a554098bd80e7d596
+  data.tar.gz: 584db18e46de8e00dede1833ab4b206d02774d3d
 SHA512:
-  metadata.gz: 031f8b26a20332034abf3db0fba1291474c96f07382bc304d358173342977bf1777b901c5a50e0c288f09331d97f68e9e9b9b3d5a69ea0aa0fe1c951ac84856c
-  data.tar.gz: 8c8e72f27a1662c9e5f305f86fd4824aae43493932832420e808183d052c1fcc233e8956ecd4d5f190d3ce34d107f475236e4dd1390d48ed96069268cd495bb9
+  metadata.gz: 1d7a502bcd878cb1847fb7c168fbfc8239d0e1a3050b6938b86b1311fada3e2595d16004bee2025166e086f90b9bb61dd77c8c8c3f1113cbceb0b6aa89ddf0d2
+  data.tar.gz: e863eb81db7bde65539315d19f14585d54f083f66524a25051e1c7f0205728fb8aec346824e14cc503453ff5b219c0da167d68e95fe2f70774b8fbeafc2f7d20

data/lib/daemon/check_thread.rb

@@ -3,6 +3,8 @@ class CheckThread < BaseThread
     $log.debug("CheckThread: Run stotage check for #{storage_name}")
     storage = $storages.detect{|s| s.name == storage_name}
     if File.writable?(storage.path)
+      storage.size_limit = storage.get_real_size if storage.auto_size?
+      # also saves size_limit
       storage.update_check_time
     else 
       error "Storage #{storage.name} with path #{storage.path} not writable"

data/lib/fc/db.rb

@@ -90,11 +90,11 @@ module FC
     def self.close
       if @options[:multi_threads]
         if @connects[Thread.current.object_id]
-          @connects[Thread.current.object_id].close
+          @connects[Thread.current.object_id].close if @connects[Thread.current.object_id]
           @connects.delete(Thread.current.object_id)
         end
       else
-        @connects.first[1].close
+        @connects.first[1].close if @connects.first
         @connects.clear
       end
     end
@@ -329,5 +329,10 @@ module FC
       FC::DB.query("ALTER TABLE #{@prefix}items MODIFY COLUMN status ENUM('new', 'ready', 'error', 'delete', 'deferred_delete') NOT NULL DEFAULT 'new'")
       FC::DB.query("ALTER TABLE #{@prefix}policies ADD COLUMN delete_deferred_time int NOT NULL DEFAULT 0")
     end
+
+    def self.migrate_4
+      FC::DB.query("ALTER TABLE #{@prefix}storages ADD COLUMN auto_size bigint(20) DEFAULT 0")
+    end
+
   end
 end

data/lib/fc/storage.rb

@@ -4,7 +4,7 @@ require 'fileutils'
 
 module FC
   class Storage < DbBase
-    set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight'
+    set_table :storages, 'name, host, dc, path, url, size, size_limit, check_time, copy_storages, url_weight, write_weight, auto_size'
     
     class << self
       attr_accessor :check_time_limit, :storages_cache_time, :get_copy_storages_mutex
@@ -39,7 +39,11 @@ module FC
       end
       super params
     end
-    
+
+    def auto_size?
+      (auto_size || 0) > 0
+    end
+
     def free
       size_limit - size
     end
@@ -52,7 +56,28 @@ module FC
       rate = free.to_f / size_limit
       rate < 0 ? 0.0 : rate
     end
-    
+
+    def size_in_status(status)
+      FC::DB.query(%{SELECT sum(i.size) as isize 
+        FROM #{FC::ItemStorage.table_name} its 
+        join #{FC::Item.table_name} i on i.id = its.item_id 
+        where its.storage_name = '#{self.name}' 
+        and its.status = '#{status}'}).first['isize'].to_i
+    end
+
+    def get_real_size
+      size_in_copy = size_in_status 'copy'
+      self.size_limit = get_disk_free_space - size_in_copy - auto_size + size
+    end
+
+    def get_disk_free_space
+      cmd = "df #{self.path.shellescape}"
+      cmd = "ssh -q -oBatchMode=yes -oStrictHostKeyChecking=no #{self.host} \"df #{self.path.shellescape}\"" unless self.class.curr_host == host 
+      r = `#{cmd} 2>&1`
+      raise r if $?.exitstatus != 0
+      r.split("\n").last.split(/\s+/)[3].to_i * 1024
+    end
+
     def get_copy_storages
       self.class.get_copy_storages_mutex.synchronize do
         unless @copy_storages_cache && Time.new.to_i - @get_copy_storages_time.to_i < self.class.storages_cache_time

data/lib/fc/version.rb

@@ -1,3 +1,3 @@
 module FC
-  VERSION = '0.5.4'.freeze
+  VERSION = '0.5.5'.freeze
 end

data/lib/manage/storages.rb

@@ -30,6 +30,7 @@ def storages_show
   Size:           #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
   Free:           #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%) 
   Size limit:     #{size_to_human storage.size_limit}
+  Size type:      #{storage.auto_size? ? "Auto (min #{ size_to_human storage.auto_size })" : 'Static'}
   Copy storages:  #{storage.copy_storages}
   Check time:     #{storage.check_time ? "#{Time.at(storage.check_time)} (#{storage.check_time_delay} seconds ago)" : ''}
   Status:         #{storage.up? ? colorize_string('UP', :green) : colorize_string('DOWN', :red)}
@@ -46,21 +47,34 @@ def storages_add
   url = stdin_read_val('Url')
   url_weight = stdin_read_val('URL weight', true).to_i
   write_weight = stdin_read_val('Write weight', true).to_i
-  size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
+  is_auto_size = %(y yes).include?(stdin_read_val('Auto size (y/n)?').downcase)
+  if is_auto_size
+    auto_size = human_to_size stdin_read_val('Minimal free disk space') {|val| "Minimal free disk space not is valid size." unless human_to_size(val) || human_to_size(val) < 1 }
+    size_limit = 0
+  else
+    auto_size = 0
+    size_limit = human_to_size stdin_read_val('Size limit') {|val| "Size limit not is valid size." unless human_to_size(val)}
+  end
+
   copy_storages = stdin_read_val('Copy storages', true)
   storages = FC::Storage.where.map(&:name)
   copy_storages = copy_storages.split(',').select{|s| storages.member?(s.strip)}.join(',').strip
   begin
     path = path +'/' unless path[-1] == '/'
     path = '/' + path unless path[0] == '/'
-    storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight)
-    print "Calc current size.. "
+    storage = FC::Storage.new(:name => name, :dc => dc, :host => host, :path => path, :url => url, :size_limit => size_limit, :copy_storages => copy_storages, :url_weight => url_weight, :write_weight => write_weight, :auto_size => auto_size)
+    print 'Calc current size.. '
     size = storage.file_size('', true)
     puts "ok"
   rescue Exception => e
     puts "Error: #{e.message}"
     exit
   end
+
+  if storage.auto_size?
+    storage.size = size
+    size_limit = storage.get_real_size
+  end
   free = size_limit - size
   puts %Q{\nStorage
   Name:         #{name}
@@ -72,6 +86,7 @@ def storages_add
   Write weight: #{write_weight}
   Size:         #{size_to_human size} (#{(size.to_f*100 / size_limit).to_i}%)
   Free:         #{size_to_human free} (#{(free.to_f*100 / size_limit).to_i}%)
+  Size type:    #{storage.auto_size? ? "Auto (min #{ size_to_human(auto_size) })" : 'Static' }
   Size limit:   #{size_to_human size_limit}
   Copy storages #{copy_storages}}
   s = Readline.readline("Continue? (y/n) ", false).strip.downcase
@@ -129,7 +144,14 @@ def storages_change
     url = stdin_read_val("Url (now #{storage.url})", true)
     url_weight = stdin_read_val("URL weight (now #{storage.url_weight})", true)
     write_weight = stdin_read_val("Write weight (now #{storage.write_weight})", true)
-    size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
+    is_auto_size = %(y yes).include?(stdin_read_val("Auto size (now #{storage.auto_size? ? 'yes' : 'no'})", true, storage.auto_size? ? 'yes' : 'no').downcase)
+    if is_auto_size
+      auto_size = human_to_size stdin_read_val("Minimal free disk space (now #{size_to_human(storage.auto_size)})", true, size_to_human(storage.auto_size)) {|val| "Minimal free disk space not is valid size." if !human_to_size(val) || human_to_size(val) < 1}
+      size_limit = 0
+    else
+      auto_size = 0
+      size_limit = stdin_read_val("Size (now #{size_to_human(storage.size_limit)})", true) {|val| "Size limit not is valid size." if !val.empty? && !human_to_size(val)}
+    end
     copy_storages = stdin_read_val("Copy storages (now #{storage.copy_storages})", true)
     
     storage.dc = dc unless dc.empty?
@@ -142,6 +164,8 @@ def storages_change
       storage.size = storage.file_size('', true)
       puts "ok"
     end
+    storage.auto_size = auto_size
+    size_limit = size_to_human(storage.get_real_size) if storage.auto_size?
     storage.url = url unless url.empty?
     storage.url_weight = url_weight.to_i unless url_weight.empty?
     storage.write_weight = write_weight.to_i unless write_weight.empty?
@@ -159,7 +183,8 @@ def storages_change
     Write weight:  #{storage.write_weight}
     Size:          #{size_to_human storage.size} (#{(storage.size_rate*100).to_i}%)
     Free:          #{size_to_human storage.free} (#{(storage.free_rate*100).to_i}%)
-    Size limit:    #{size_to_human storage.size_limit}
+    Size type:     #{storage.auto_size? ? "Auto (Min #{size_to_human auto_size})" : 'Static' }
+    Size limit:    #{size_to_human storage.size_limit }
     Copy storages: #{storage.copy_storages}}
     s = Readline.readline("Continue? (y/n) ", false).strip.downcase
     puts ""

data/lib/utils.rb

@@ -47,11 +47,12 @@ def human_to_size(size)
   result.to_i
 end
 
-def stdin_read_val(name, can_empty = false)
+def stdin_read_val(name, can_empty = false, default_value = nil)
   while val = Readline.readline("#{name}: ", false).strip.downcase
     if val.empty? && !can_empty
       puts "Input non empty #{name}."
-    else 
+    else
+      val = default_value if default_value && val.empty?
       if block_given?
         if err = yield(val) 
           puts err

data/test/storage_test.rb

@@ -8,6 +8,7 @@ class StorageTest < Test::Unit::TestCase
       @@storages << FC::Storage.new(:name => 'rec2-sda', :host => 'rec2', :size => 0, :copy_storages => 'rec1-sda,rec3-sda', :size_limit => 100)
       @@storages << FC::Storage.new(:name => 'rec3-sda', :host => 'rec3', :size => 8, :copy_storages => 'rec1-sda,rec2-sda', :size_limit => 10)
       @@storages << FC::Storage.new(:name => 'rec2-sdb', :host => 'rec2', :size => 0, :size_limit => 100)
+      @@storages << FC::Storage.new(:name => 'rec1-sdc', :host => 'rec1', :size => 100, :auto_size => 50)
       @@storages.each {|storage| storage.save}
       
       @@policy = FC::Policy.new(:create_storages => 'rec1-sda,rec2-sda,rec2-sdb', :copies => 1, :name => 'policy 1')
@@ -15,6 +16,8 @@ class StorageTest < Test::Unit::TestCase
     end
     def shutdown
       FC::DB.query("DELETE FROM policies")
+      FC::DB.query("DELETE FROM items_storages")
+      FC::DB.query("DELETE FROM items")
       FC::DB.query("DELETE FROM storages")
     end
   end 
@@ -79,4 +82,29 @@ class StorageTest < Test::Unit::TestCase
     assert_equal 9.to_f/10, storage.free_rate, 'first storage free_rate must be free/size_limit' 
     assert_equal 1.to_f/10, storage.size_rate, 'first storage free must be size/size_limit'
   end
+
+  should 'size_in_status copy' do
+    # create fc_item_storage in status 'copy'
+    @item = FC::Item.new(:name => '/test item', :policy_id => 1, :size => 100)
+    @item.save
+    @item_storage = FC::ItemStorage.new(:item_id => @item.id, :storage_name => @@storages[0].name, :status => 'copy')
+    @item_storage.save
+    size_in_copy_status = @@storages[0].size_in_status 'copy'
+    assert_equal 100, size_in_copy_status, 'items sum size in status copy must be correct'
+  end
+
+  should 'get real size with items status in copy' do # most worst case
+    disk_free = 1024
+    item_size = 100
+
+    # mock real disk free method
+    @@storages[4].define_singleton_method(:get_disk_free_space) do
+        disk_free
+    end
+    @item = FC::Item.new(:name => '/test item rec1-sdc', :policy_id => 1, :size => item_size)
+    @item.save
+    @item_storage = FC::ItemStorage.new(:item_id => @item.id, :storage_name => @@storages[4].name, :status => 'copy')
+    @item_storage.save
+    assert_equal disk_free - item_size - @@storages[4].auto_size + @@storages[4].size, @@storages[4].get_real_size, 'storage size must be correct with items in copy status'
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filecluster
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - sh
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-14 00:00:00.000000000 Z
+date: 2017-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mysql2